[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[JoshRosen]

Github user JoshRosen commented on a diff in the pull request:

https://github.com/apache/spark/pull/5694#discussion_r32804762
  
--- Diff: dev/run-tests.py ---
@@ -0,0 +1,536 @@
+#!/usr/bin/env python2
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+import re
+import sys
+import shutil
+import subprocess
+from collections import namedtuple
+
+SPARK_HOME = os.path.join(os.path.dirname(os.path.realpath(__file__)), 
"..")
+USER_HOME = os.environ.get("HOME")
+
+
+def get_error_codes(err_code_file):
+"""Function to retrieve all block numbers from the `run-tests-codes.sh`
+file to maintain backwards compatibility with the `run-tests-jenkins`
+script"""
+
+with open(err_code_file, 'r') as f:
+err_codes = [e.split()[1].strip().split('=')
+ for e in f if e.startswith("readonly")]
+return dict(err_codes)
+
+
+ERROR_CODES = get_error_codes(os.path.join(SPARK_HOME, 
"dev/run-tests-codes.sh"))
+
+
+def exit_from_command_with_retcode(cmd, retcode):
+print "[error] running", cmd, "; received return code", retcode
+sys.exit(int(os.environ.get("CURRENT_BLOCK", 255)))
+
+
+def rm_r(path):
+"""Given an arbitrary path properly remove it with the correct python
+construct if it exists
+- from: http://stackoverflow.com/a/9559881""";
+
+if os.path.isdir(path):
+shutil.rmtree(path)
+elif os.path.exists(path):
+os.remove(path)
+
+
+def run_cmd(cmd):
+"""Given a command as a list of arguments will attempt to execute the
+command from the determined SPARK_HOME directory and, on failure, print
+an error message"""
+
+if not isinstance(cmd, list):
+cmd = cmd.split()
+try:
+subprocess.check_call(cmd)
+except subprocess.CalledProcessError as e:
+exit_from_command_with_retcode(e.cmd, e.returncode)
+
+
+def is_exe(path):
+"""Check if a given path is an executable file
+- from: http://stackoverflow.com/a/377028""";
+
+return os.path.isfile(path) and os.access(path, os.X_OK)
+
+
+def which(program):
+"""Find and return the given program by its absolute path or 'None'
+- from: http://stackoverflow.com/a/377028""";
+
+fpath, fname = os.path.split(program)
+
+if fpath:
+if is_exe(program):
+return program
+else:
+for path in os.environ.get("PATH").split(os.pathsep):
+path = path.strip('"')
+exe_file = os.path.join(path, program)
+if is_exe(exe_file):
+return exe_file
+return None
+
+
+def determine_java_executable():
+"""Will return the path of the java executable that will be used by 
Spark's
+tests or `None`"""
+
+# Any changes in the way that Spark's build detects java must be 
reflected
+# here. Currently the build looks for $JAVA_HOME/bin/java then falls 
back to
+# the `java` executable on the path
+
+java_home = os.environ.get("JAVA_HOME")
+
+# check if there is an executable at $JAVA_HOME/bin/java
+java_exe = which(os.path.join(java_home, "bin", "java")) if java_home 
else None
+# if the java_exe wasn't set, check for a `java` version on the $PATH
+return java_exe if java_exe else which("java")
+
+
+JavaVersion = namedtuple('JavaVersion', ['major', 'minor', 'patch', 
'update'])
+
+
+def determine_java_version(java_exe):
+"""Given a valid java executable will return its version in named 
tuple format
+with accessors '.major', '.minor', '.patch', '.update'"""
+
+raw_output = subprocess.check_output([java_exe, "-version"],
+ stderr=subprocess.STDOUT)
+raw_version_str = raw_output.split('\n')[0]  # eg 'java version 
"1.8.0_25"'
+v

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[JoshRosen]

Github user JoshRosen commented on a diff in the pull request:

https://github.com/apache/spark/pull/5694#discussion_r32697343
  
--- Diff: dev/run-tests.py ---
@@ -0,0 +1,536 @@
+#!/usr/bin/env python2
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+import re
+import sys
+import shutil
+import subprocess
+from collections import namedtuple
+
+SPARK_HOME = os.path.join(os.path.dirname(os.path.realpath(__file__)), 
"..")
+USER_HOME = os.environ.get("HOME")
+
+
+def get_error_codes(err_code_file):
+"""Function to retrieve all block numbers from the `run-tests-codes.sh`
+file to maintain backwards compatibility with the `run-tests-jenkins`
+script"""
+
+with open(err_code_file, 'r') as f:
+err_codes = [e.split()[1].strip().split('=')
+ for e in f if e.startswith("readonly")]
+return dict(err_codes)
+
+
+ERROR_CODES = get_error_codes(os.path.join(SPARK_HOME, 
"dev/run-tests-codes.sh"))
+
+
+def exit_from_command_with_retcode(cmd, retcode):
+print "[error] running", cmd, "; received return code", retcode
+sys.exit(int(os.environ.get("CURRENT_BLOCK", 255)))
+
+
+def rm_r(path):
+"""Given an arbitrary path properly remove it with the correct python
+construct if it exists
+- from: http://stackoverflow.com/a/9559881""";
+
+if os.path.isdir(path):
+shutil.rmtree(path)
+elif os.path.exists(path):
+os.remove(path)
+
+
+def run_cmd(cmd):
+"""Given a command as a list of arguments will attempt to execute the
+command from the determined SPARK_HOME directory and, on failure, print
+an error message"""
+
+if not isinstance(cmd, list):
+cmd = cmd.split()
+try:
+subprocess.check_call(cmd)
+except subprocess.CalledProcessError as e:
+exit_from_command_with_retcode(e.cmd, e.returncode)
+
+
+def is_exe(path):
+"""Check if a given path is an executable file
+- from: http://stackoverflow.com/a/377028""";
+
+return os.path.isfile(path) and os.access(path, os.X_OK)
+
+
+def which(program):
+"""Find and return the given program by its absolute path or 'None'
+- from: http://stackoverflow.com/a/377028""";
+
+fpath, fname = os.path.split(program)
+
+if fpath:
+if is_exe(program):
+return program
+else:
+for path in os.environ.get("PATH").split(os.pathsep):
+path = path.strip('"')
+exe_file = os.path.join(path, program)
+if is_exe(exe_file):
+return exe_file
+return None
+
+
+def determine_java_executable():
+"""Will return the path of the java executable that will be used by 
Spark's
+tests or `None`"""
+
+# Any changes in the way that Spark's build detects java must be 
reflected
+# here. Currently the build looks for $JAVA_HOME/bin/java then falls 
back to
+# the `java` executable on the path
+
+java_home = os.environ.get("JAVA_HOME")
+
+# check if there is an executable at $JAVA_HOME/bin/java
+java_exe = which(os.path.join(java_home, "bin", "java")) if java_home 
else None
+# if the java_exe wasn't set, check for a `java` version on the $PATH
+return java_exe if java_exe else which("java")
+
+
+JavaVersion = namedtuple('JavaVersion', ['major', 'minor', 'patch', 
'update'])
+
+
+def determine_java_version(java_exe):
+"""Given a valid java executable will return its version in named 
tuple format
+with accessors '.major', '.minor', '.patch', '.update'"""
+
+raw_output = subprocess.check_output([java_exe, "-version"],
+ stderr=subprocess.STDOUT)
+raw_version_str = raw_output.split('\n')[0]  # eg 'java version 
"1.8.0_25"'
+v

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[brennonyork]

Github user brennonyork commented on a diff in the pull request:

https://github.com/apache/spark/pull/5694#discussion_r32694170
  
--- Diff: dev/run-tests.py ---
@@ -0,0 +1,536 @@
+#!/usr/bin/env python2
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+import re
+import sys
+import shutil
+import subprocess
+from collections import namedtuple
+
+SPARK_HOME = os.path.join(os.path.dirname(os.path.realpath(__file__)), 
"..")
+USER_HOME = os.environ.get("HOME")
+
+
+def get_error_codes(err_code_file):
+"""Function to retrieve all block numbers from the `run-tests-codes.sh`
+file to maintain backwards compatibility with the `run-tests-jenkins`
+script"""
+
+with open(err_code_file, 'r') as f:
+err_codes = [e.split()[1].strip().split('=')
+ for e in f if e.startswith("readonly")]
+return dict(err_codes)
+
+
+ERROR_CODES = get_error_codes(os.path.join(SPARK_HOME, 
"dev/run-tests-codes.sh"))
+
+
+def exit_from_command_with_retcode(cmd, retcode):
+print "[error] running", cmd, "; received return code", retcode
+sys.exit(int(os.environ.get("CURRENT_BLOCK", 255)))
+
+
+def rm_r(path):
+"""Given an arbitrary path properly remove it with the correct python
+construct if it exists
+- from: http://stackoverflow.com/a/9559881""";
+
+if os.path.isdir(path):
+shutil.rmtree(path)
+elif os.path.exists(path):
+os.remove(path)
+
+
+def run_cmd(cmd):
+"""Given a command as a list of arguments will attempt to execute the
+command from the determined SPARK_HOME directory and, on failure, print
+an error message"""
+
+if not isinstance(cmd, list):
+cmd = cmd.split()
+try:
+subprocess.check_call(cmd)
+except subprocess.CalledProcessError as e:
+exit_from_command_with_retcode(e.cmd, e.returncode)
+
+
+def is_exe(path):
+"""Check if a given path is an executable file
+- from: http://stackoverflow.com/a/377028""";
+
+return os.path.isfile(path) and os.access(path, os.X_OK)
+
+
+def which(program):
+"""Find and return the given program by its absolute path or 'None'
+- from: http://stackoverflow.com/a/377028""";
+
+fpath, fname = os.path.split(program)
+
+if fpath:
+if is_exe(program):
+return program
+else:
+for path in os.environ.get("PATH").split(os.pathsep):
+path = path.strip('"')
+exe_file = os.path.join(path, program)
+if is_exe(exe_file):
+return exe_file
+return None
+
+
+def determine_java_executable():
+"""Will return the path of the java executable that will be used by 
Spark's
+tests or `None`"""
+
+# Any changes in the way that Spark's build detects java must be 
reflected
+# here. Currently the build looks for $JAVA_HOME/bin/java then falls 
back to
+# the `java` executable on the path
+
+java_home = os.environ.get("JAVA_HOME")
+
+# check if there is an executable at $JAVA_HOME/bin/java
+java_exe = which(os.path.join(java_home, "bin", "java")) if java_home 
else None
+# if the java_exe wasn't set, check for a `java` version on the $PATH
+return java_exe if java_exe else which("java")
+
+
+JavaVersion = namedtuple('JavaVersion', ['major', 'minor', 'patch', 
'update'])
+
+
+def determine_java_version(java_exe):
+"""Given a valid java executable will return its version in named 
tuple format
+with accessors '.major', '.minor', '.patch', '.update'"""
+
+raw_output = subprocess.check_output([java_exe, "-version"],
+ stderr=subprocess.STDOUT)
+raw_version_str = raw_output.split('\n')[0]  # eg 'java version 
"1.8.0_25"'
+   

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[JoshRosen]

Github user JoshRosen commented on a diff in the pull request:

https://github.com/apache/spark/pull/5694#discussion_r32694068
  
--- Diff: dev/run-tests.py ---
@@ -0,0 +1,536 @@
+#!/usr/bin/env python2
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+import re
+import sys
+import shutil
+import subprocess
+from collections import namedtuple
+
+SPARK_HOME = os.path.join(os.path.dirname(os.path.realpath(__file__)), 
"..")
+USER_HOME = os.environ.get("HOME")
+
+
+def get_error_codes(err_code_file):
+"""Function to retrieve all block numbers from the `run-tests-codes.sh`
+file to maintain backwards compatibility with the `run-tests-jenkins`
+script"""
+
+with open(err_code_file, 'r') as f:
+err_codes = [e.split()[1].strip().split('=')
+ for e in f if e.startswith("readonly")]
+return dict(err_codes)
+
+
+ERROR_CODES = get_error_codes(os.path.join(SPARK_HOME, 
"dev/run-tests-codes.sh"))
+
+
+def exit_from_command_with_retcode(cmd, retcode):
+print "[error] running", cmd, "; received return code", retcode
+sys.exit(int(os.environ.get("CURRENT_BLOCK", 255)))
+
+
+def rm_r(path):
+"""Given an arbitrary path properly remove it with the correct python
+construct if it exists
+- from: http://stackoverflow.com/a/9559881""";
+
+if os.path.isdir(path):
+shutil.rmtree(path)
+elif os.path.exists(path):
+os.remove(path)
+
+
+def run_cmd(cmd):
+"""Given a command as a list of arguments will attempt to execute the
+command from the determined SPARK_HOME directory and, on failure, print
+an error message"""
+
+if not isinstance(cmd, list):
+cmd = cmd.split()
+try:
+subprocess.check_call(cmd)
+except subprocess.CalledProcessError as e:
+exit_from_command_with_retcode(e.cmd, e.returncode)
+
+
+def is_exe(path):
+"""Check if a given path is an executable file
+- from: http://stackoverflow.com/a/377028""";
+
+return os.path.isfile(path) and os.access(path, os.X_OK)
+
+
+def which(program):
+"""Find and return the given program by its absolute path or 'None'
+- from: http://stackoverflow.com/a/377028""";
+
+fpath, fname = os.path.split(program)
+
+if fpath:
+if is_exe(program):
+return program
+else:
+for path in os.environ.get("PATH").split(os.pathsep):
+path = path.strip('"')
+exe_file = os.path.join(path, program)
+if is_exe(exe_file):
+return exe_file
+return None
+
+
+def determine_java_executable():
+"""Will return the path of the java executable that will be used by 
Spark's
+tests or `None`"""
+
+# Any changes in the way that Spark's build detects java must be 
reflected
+# here. Currently the build looks for $JAVA_HOME/bin/java then falls 
back to
+# the `java` executable on the path
+
+java_home = os.environ.get("JAVA_HOME")
+
+# check if there is an executable at $JAVA_HOME/bin/java
+java_exe = which(os.path.join(java_home, "bin", "java")) if java_home 
else None
+# if the java_exe wasn't set, check for a `java` version on the $PATH
+return java_exe if java_exe else which("java")
+
+
+JavaVersion = namedtuple('JavaVersion', ['major', 'minor', 'patch', 
'update'])
+
+
+def determine_java_version(java_exe):
+"""Given a valid java executable will return its version in named 
tuple format
+with accessors '.major', '.minor', '.patch', '.update'"""
+
+raw_output = subprocess.check_output([java_exe, "-version"],
+ stderr=subprocess.STDOUT)
+raw_version_str = raw_output.split('\n')[0]  # eg 'java version 
"1.8.0_25"'
+v

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[JoshRosen]

Github user JoshRosen commented on a diff in the pull request:

https://github.com/apache/spark/pull/5694#discussion_r32693784
  
--- Diff: dev/run-tests.py ---
@@ -0,0 +1,536 @@
+#!/usr/bin/env python2
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+import re
+import sys
+import shutil
+import subprocess
+from collections import namedtuple
+
+SPARK_HOME = os.path.join(os.path.dirname(os.path.realpath(__file__)), 
"..")
+USER_HOME = os.environ.get("HOME")
+
+
+def get_error_codes(err_code_file):
+"""Function to retrieve all block numbers from the `run-tests-codes.sh`
+file to maintain backwards compatibility with the `run-tests-jenkins`
+script"""
+
+with open(err_code_file, 'r') as f:
+err_codes = [e.split()[1].strip().split('=')
+ for e in f if e.startswith("readonly")]
+return dict(err_codes)
+
+
+ERROR_CODES = get_error_codes(os.path.join(SPARK_HOME, 
"dev/run-tests-codes.sh"))
+
+
+def exit_from_command_with_retcode(cmd, retcode):
+print "[error] running", cmd, "; received return code", retcode
+sys.exit(int(os.environ.get("CURRENT_BLOCK", 255)))
+
+
+def rm_r(path):
+"""Given an arbitrary path properly remove it with the correct python
+construct if it exists
+- from: http://stackoverflow.com/a/9559881""";
+
+if os.path.isdir(path):
+shutil.rmtree(path)
+elif os.path.exists(path):
+os.remove(path)
+
+
+def run_cmd(cmd):
+"""Given a command as a list of arguments will attempt to execute the
+command from the determined SPARK_HOME directory and, on failure, print
+an error message"""
+
+if not isinstance(cmd, list):
+cmd = cmd.split()
+try:
+subprocess.check_call(cmd)
+except subprocess.CalledProcessError as e:
+exit_from_command_with_retcode(e.cmd, e.returncode)
+
+
+def is_exe(path):
+"""Check if a given path is an executable file
+- from: http://stackoverflow.com/a/377028""";
+
+return os.path.isfile(path) and os.access(path, os.X_OK)
+
+
+def which(program):
+"""Find and return the given program by its absolute path or 'None'
+- from: http://stackoverflow.com/a/377028""";
+
+fpath, fname = os.path.split(program)
+
+if fpath:
+if is_exe(program):
+return program
+else:
+for path in os.environ.get("PATH").split(os.pathsep):
+path = path.strip('"')
+exe_file = os.path.join(path, program)
+if is_exe(exe_file):
+return exe_file
+return None
+
+
+def determine_java_executable():
+"""Will return the path of the java executable that will be used by 
Spark's
+tests or `None`"""
+
+# Any changes in the way that Spark's build detects java must be 
reflected
+# here. Currently the build looks for $JAVA_HOME/bin/java then falls 
back to
+# the `java` executable on the path
+
+java_home = os.environ.get("JAVA_HOME")
+
+# check if there is an executable at $JAVA_HOME/bin/java
+java_exe = which(os.path.join(java_home, "bin", "java")) if java_home 
else None
+# if the java_exe wasn't set, check for a `java` version on the $PATH
+return java_exe if java_exe else which("java")
+
+
+JavaVersion = namedtuple('JavaVersion', ['major', 'minor', 'patch', 
'update'])
+
+
+def determine_java_version(java_exe):
+"""Given a valid java executable will return its version in named 
tuple format
+with accessors '.major', '.minor', '.patch', '.update'"""
+
+raw_output = subprocess.check_output([java_exe, "-version"],
+ stderr=subprocess.STDOUT)
+raw_version_str = raw_output.split('\n')[0]  # eg 'java version 
"1.8.0_25"'
+v

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[brennonyork]

Github user brennonyork commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112976517
  
Thanks for the PR merge @JoshRosen. I'll go ahead and make a hotfix branch 
to capture the last few nits you have above!


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[asfgit]

Github user asfgit closed the pull request at:

https://github.com/apache/spark/pull/5694


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[JoshRosen]

Github user JoshRosen commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112914050
  
Some minor nits aside, this looks good enough to me, so I'm going to merge 
it into master and will submit a series of followup PRs to perform more 
modularization / refactoring of the dependency graph logic.

Thanks for working on this!


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[JoshRosen]

Github user JoshRosen commented on a diff in the pull request:

https://github.com/apache/spark/pull/5694#discussion_r32661175
  
--- Diff: dev/run-tests.py ---
@@ -0,0 +1,536 @@
+#!/usr/bin/env python2
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+import re
+import sys
+import shutil
+import subprocess
+from collections import namedtuple
+
+SPARK_HOME = os.path.join(os.path.dirname(os.path.realpath(__file__)), 
"..")
+USER_HOME = os.environ.get("HOME")
+
+
+def get_error_codes(err_code_file):
+"""Function to retrieve all block numbers from the `run-tests-codes.sh`
+file to maintain backwards compatibility with the `run-tests-jenkins`
+script"""
+
+with open(err_code_file, 'r') as f:
+err_codes = [e.split()[1].strip().split('=')
+ for e in f if e.startswith("readonly")]
+return dict(err_codes)
+
+
+ERROR_CODES = get_error_codes(os.path.join(SPARK_HOME, 
"dev/run-tests-codes.sh"))
+
+
+def exit_from_command_with_retcode(cmd, retcode):
+print "[error] running", cmd, "; received return code", retcode
+sys.exit(int(os.environ.get("CURRENT_BLOCK", 255)))
+
+
+def rm_r(path):
+"""Given an arbitrary path properly remove it with the correct python
+construct if it exists
+- from: http://stackoverflow.com/a/9559881""";
+
+if os.path.isdir(path):
+shutil.rmtree(path)
+elif os.path.exists(path):
+os.remove(path)
+
+
+def run_cmd(cmd):
+"""Given a command as a list of arguments will attempt to execute the
+command from the determined SPARK_HOME directory and, on failure, print
+an error message"""
+
+if not isinstance(cmd, list):
+cmd = cmd.split()
+try:
+subprocess.check_call(cmd)
+except subprocess.CalledProcessError as e:
+exit_from_command_with_retcode(e.cmd, e.returncode)
+
+
+def is_exe(path):
+"""Check if a given path is an executable file
+- from: http://stackoverflow.com/a/377028""";
+
+return os.path.isfile(path) and os.access(path, os.X_OK)
+
+
+def which(program):
+"""Find and return the given program by its absolute path or 'None'
+- from: http://stackoverflow.com/a/377028""";
+
+fpath, fname = os.path.split(program)
+
+if fpath:
+if is_exe(program):
+return program
+else:
+for path in os.environ.get("PATH").split(os.pathsep):
+path = path.strip('"')
+exe_file = os.path.join(path, program)
+if is_exe(exe_file):
+return exe_file
+return None
+
+
+def determine_java_executable():
+"""Will return the path of the java executable that will be used by 
Spark's
+tests or `None`"""
+
+# Any changes in the way that Spark's build detects java must be 
reflected
+# here. Currently the build looks for $JAVA_HOME/bin/java then falls 
back to
+# the `java` executable on the path
+
+java_home = os.environ.get("JAVA_HOME")
+
+# check if there is an executable at $JAVA_HOME/bin/java
+java_exe = which(os.path.join(java_home, "bin", "java")) if java_home 
else None
+# if the java_exe wasn't set, check for a `java` version on the $PATH
+return java_exe if java_exe else which("java")
+
+
+JavaVersion = namedtuple('JavaVersion', ['major', 'minor', 'patch', 
'update'])
+
+
+def determine_java_version(java_exe):
+"""Given a valid java executable will return its version in named 
tuple format
+with accessors '.major', '.minor', '.patch', '.update'"""
+
+raw_output = subprocess.check_output([java_exe, "-version"],
+ stderr=subprocess.STDOUT)
+raw_version_str = raw_output.split('\n')[0]  # eg 'java version 
"1.8.0_25"'
+v

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[JoshRosen]

Github user JoshRosen commented on a diff in the pull request:

https://github.com/apache/spark/pull/5694#discussion_r32660292
  
--- Diff: dev/run-tests.py ---
@@ -0,0 +1,536 @@
+#!/usr/bin/env python2
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+import re
+import sys
+import shutil
+import subprocess
+from collections import namedtuple
+
+SPARK_HOME = os.path.join(os.path.dirname(os.path.realpath(__file__)), 
"..")
+USER_HOME = os.environ.get("HOME")
+
+
+def get_error_codes(err_code_file):
+"""Function to retrieve all block numbers from the `run-tests-codes.sh`
+file to maintain backwards compatibility with the `run-tests-jenkins`
+script"""
+
+with open(err_code_file, 'r') as f:
+err_codes = [e.split()[1].strip().split('=')
+ for e in f if e.startswith("readonly")]
+return dict(err_codes)
+
+
+ERROR_CODES = get_error_codes(os.path.join(SPARK_HOME, 
"dev/run-tests-codes.sh"))
+
+
+def exit_from_command_with_retcode(cmd, retcode):
+print "[error] running", cmd, "; received return code", retcode
--- End diff --

Minor nit / annoyance here: this ends up printing things like

```
[error] running ['/Users/joshrosen/Documents/Spark/dev/../build/mvn', 
'-Pyarn', '-Phadoop-2.3', '-Dhadoop.version=2.3.0', '-Pkinesis-asl', '-Phive', 
'-Phive-thriftserver', 'clean', 'package', '-DskipTests'] ; received return 
code 1
```

which makes it hard to copy and paste the command to run it manually in the 
shell.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[JoshRosen]

Github user JoshRosen commented on a diff in the pull request:

https://github.com/apache/spark/pull/5694#discussion_r32658498
  
--- Diff: dev/run-tests.py ---
@@ -0,0 +1,536 @@
+#!/usr/bin/env python2
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+import re
+import sys
+import shutil
+import subprocess
+from collections import namedtuple
+
+SPARK_HOME = os.path.join(os.path.dirname(os.path.realpath(__file__)), 
"..")
+USER_HOME = os.environ.get("HOME")
+
+
+def get_error_codes(err_code_file):
+"""Function to retrieve all block numbers from the `run-tests-codes.sh`
+file to maintain backwards compatibility with the `run-tests-jenkins`
+script"""
+
+with open(err_code_file, 'r') as f:
+err_codes = [e.split()[1].strip().split('=')
+ for e in f if e.startswith("readonly")]
+return dict(err_codes)
+
+
+ERROR_CODES = get_error_codes(os.path.join(SPARK_HOME, 
"dev/run-tests-codes.sh"))
+
+
+def exit_from_command_with_retcode(cmd, retcode):
+print "[error] running", cmd, "; received return code", retcode
+sys.exit(int(os.environ.get("CURRENT_BLOCK", 255)))
+
+
+def rm_r(path):
+"""Given an arbitrary path properly remove it with the correct python
+construct if it exists
+- from: http://stackoverflow.com/a/9559881""";
+
+if os.path.isdir(path):
+shutil.rmtree(path)
+elif os.path.exists(path):
+os.remove(path)
+
+
+def run_cmd(cmd):
+"""Given a command as a list of arguments will attempt to execute the
+command from the determined SPARK_HOME directory and, on failure, print
+an error message"""
+
+if not isinstance(cmd, list):
+cmd = cmd.split()
+try:
+subprocess.check_call(cmd)
+except subprocess.CalledProcessError as e:
+exit_from_command_with_retcode(e.cmd, e.returncode)
+
+
+def is_exe(path):
+"""Check if a given path is an executable file
+- from: http://stackoverflow.com/a/377028""";
+
+return os.path.isfile(path) and os.access(path, os.X_OK)
+
+
+def which(program):
+"""Find and return the given program by its absolute path or 'None'
+- from: http://stackoverflow.com/a/377028""";
+
+fpath, fname = os.path.split(program)
+
+if fpath:
+if is_exe(program):
+return program
+else:
+for path in os.environ.get("PATH").split(os.pathsep):
+path = path.strip('"')
+exe_file = os.path.join(path, program)
+if is_exe(exe_file):
+return exe_file
+return None
+
+
+def determine_java_executable():
+"""Will return the path of the java executable that will be used by 
Spark's
+tests or `None`"""
+
+# Any changes in the way that Spark's build detects java must be 
reflected
+# here. Currently the build looks for $JAVA_HOME/bin/java then falls 
back to
+# the `java` executable on the path
+
+java_home = os.environ.get("JAVA_HOME")
+
+# check if there is an executable at $JAVA_HOME/bin/java
+java_exe = which(os.path.join(java_home, "bin", "java")) if java_home 
else None
+# if the java_exe wasn't set, check for a `java` version on the $PATH
+return java_exe if java_exe else which("java")
+
+
+JavaVersion = namedtuple('JavaVersion', ['major', 'minor', 'patch', 
'update'])
+
+
+def determine_java_version(java_exe):
+"""Given a valid java executable will return its version in named 
tuple format
+with accessors '.major', '.minor', '.patch', '.update'"""
+
+raw_output = subprocess.check_output([java_exe, "-version"],
+ stderr=subprocess.STDOUT)
+raw_version_str = raw_output.split('\n')[0]  # eg 'java version 
"1.8.0_25"'
+v

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[AmplabJenkins]

Github user AmplabJenkins commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112617286
  
Merged build finished. Test PASSed.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[SparkQA]

Github user SparkQA commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112617265
  
  [Test build #35010 has 
finished](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/35010/console)
 for   PR 5694 at commit 
[`154ed73`](https://github.com/apache/spark/commit/154ed739026af964ab38e564abdf91124a9acf96).
 * This patch **passes all tests**.
 * This patch merges cleanly.
 * This patch adds no public classes.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[AmplabJenkins]

Github user AmplabJenkins commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112604359
  
Merged build finished. Test PASSed.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[SparkQA]

Github user SparkQA commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112604329
  
  [Test build #35009 has 
finished](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/35009/console)
 for   PR 5694 at commit 
[`154ed73`](https://github.com/apache/spark/commit/154ed739026af964ab38e564abdf91124a9acf96).
 * This patch **passes all tests**.
 * This patch merges cleanly.
 * This patch adds no public classes.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[SparkQA]

Github user SparkQA commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112600659
  
  [Test build #35010 has 
started](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/35010/consoleFull)
 for   PR 5694 at commit 
[`154ed73`](https://github.com/apache/spark/commit/154ed739026af964ab38e564abdf91124a9acf96).


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[AmplabJenkins]

Github user AmplabJenkins commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112600313
  
Merged build started.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[AmplabJenkins]

Github user AmplabJenkins commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112600302
  
 Merged build triggered.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[brennonyork]

Github user brennonyork commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112599782
  
jenkins, retest this please


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[SparkQA]

Github user SparkQA commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112596495
  
  [Test build #35008 has 
finished](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/35008/console)
 for   PR 5694 at commit 
[`3922a85`](https://github.com/apache/spark/commit/3922a85d68a4936a7d2b125b1850f87553a7e537).
 * This patch **fails Spark unit tests**.
 * This patch merges cleanly.
 * This patch adds no public classes.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[AmplabJenkins]

Github user AmplabJenkins commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112596521
  
Merged build finished. Test FAILed.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[SparkQA]

Github user SparkQA commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112581126
  
  [Test build #35009 has 
started](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/35009/consoleFull)
 for   PR 5694 at commit 
[`154ed73`](https://github.com/apache/spark/commit/154ed739026af964ab38e564abdf91124a9acf96).


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[AmplabJenkins]

Github user AmplabJenkins commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112580609
  
Merged build started.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[AmplabJenkins]

Github user AmplabJenkins commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112580554
  
 Merged build triggered.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[brennonyork]

Github user brennonyork commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112578835
  
Roger. I see it now. Will have a fix up shortly.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[JoshRosen]

Github user JoshRosen commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112577081
  
@brennonyork Take a look at the stacktrace in my comment: if the `JAVA_HOME 
` environment variable is not defined, then we attempt to call `.endswith` on 
`None`, leading to an error that prevents us from just falling back to `java`.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[SparkQA]

Github user SparkQA commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112576416
  
  [Test build #35008 has 
started](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/35008/consoleFull)
 for   PR 5694 at commit 
[`3922a85`](https://github.com/apache/spark/commit/3922a85d68a4936a7d2b125b1850f87553a7e537).


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[AmplabJenkins]

Github user AmplabJenkins commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112576230
  
Merged build started.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[AmplabJenkins]

Github user AmplabJenkins commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112576209
  
 Merged build triggered.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[brennonyork]

Github user brennonyork commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112575903
  
@JoshRosen for the `JAVA_HOME` issue, are you asking if the code checks the 
regular `PATH` for a `java` executable after checking for `JAVA_HOME`? I 
believe what you're asking is already done 
[here](https://github.com/brennonyork/spark/blob/SPARK-7017/dev/run-tests.py#L112).


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[JoshRosen]

Github user JoshRosen commented on a diff in the pull request:

https://github.com/apache/spark/pull/5694#discussion_r32570777
  
--- Diff: dev/run-tests.py ---
@@ -0,0 +1,536 @@
+#!/usr/bin/env python2
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+import re
+import sys
+import shutil
+import subprocess
+from collections import namedtuple
+
+SPARK_HOME = os.path.join(os.path.dirname(os.path.realpath(__file__)), 
"..")
+USER_HOME = os.environ.get("HOME")
+
+
+def get_error_codes(err_code_file):
+"""Function to retrieve all block numbers from the `run-tests-codes.sh`
+file to maintain backwards compatibility with the `run-tests-jenkins`
+script"""
+
+with open(err_code_file, 'r') as f:
+err_codes = [e.split()[1].strip().split('=')
+ for e in f if e.startswith("readonly")]
+return dict(err_codes)
+
+
+ERROR_CODES = get_error_codes(os.path.join(SPARK_HOME, 
"dev/run-tests-codes.sh"))
+
+
+def exit_from_command_with_retcode(cmd, retcode):
+print "[error] running", cmd, "; received return code", retcode
+sys.exit(int(os.environ.get("CURRENT_BLOCK", 255)))
+
+
+def rm_r(path):
+"""Given an arbitrary path properly remove it with the correct python
+construct if it exists
+- from: http://stackoverflow.com/a/9559881""";
+
+if os.path.isdir(path):
+shutil.rmtree(path)
+elif os.path.exists(path):
+os.remove(path)
+
+
+def run_cmd(cmd):
+"""Given a command as a list of arguments will attempt to execute the
+command from the determined SPARK_HOME directory and, on failure, print
+an error message"""
+
+if not isinstance(cmd, list):
+cmd = cmd.split()
+try:
+subprocess.check_call(cmd)
+except subprocess.CalledProcessError as e:
+exit_from_command_with_retcode(e.cmd, e.returncode)
+
+
+def is_exe(path):
+"""Check if a given path is an executable file
+- from: http://stackoverflow.com/a/377028""";
+
+return os.path.isfile(path) and os.access(path, os.X_OK)
+
+
+def which(program):
+"""Find and return the given program by its absolute path or 'None'
+- from: http://stackoverflow.com/a/377028""";
+
+fpath, fname = os.path.split(program)
+
+if fpath:
+if is_exe(program):
+return program
+else:
+for path in os.environ.get("PATH").split(os.pathsep):
+path = path.strip('"')
+exe_file = os.path.join(path, program)
+if is_exe(exe_file):
+return exe_file
+return None
+
+
+def determine_java_executable():
+"""Will return the path of the java executable that will be used by 
Spark's
+tests or `None`"""
+
+# Any changes in the way that Spark's build detects java must be 
reflected
+# here. Currently the build looks for $JAVA_HOME/bin/java then falls 
back to
+# the `java` executable on the path
+
+java_home = os.environ.get("JAVA_HOME")
+
+# check if there is an executable at $JAVA_HOME/bin/java
+java_exe = which(os.path.join(java_home, "bin", "java"))
+# if the java_exe wasn't set, check for a `java` version on the $PATH
+return java_exe if java_exe else which("java")
+
+
+JavaVersion = namedtuple('JavaVersion', ['major', 'minor', 'patch', 
'update'])
+
+
+def determine_java_version(java_exe):
+"""Given a valid java executable will return its version in named 
tuple format
+with accessors '.major', '.minor', '.patch', '.update'"""
+
+raw_output = subprocess.check_output([java_exe, "-version"],
+ stderr=subprocess.STDOUT)
+raw_version_str = raw_output.split('\n')[0]  # eg 'java version 
"1.8.0_25"'
+version_str = raw_version

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[JoshRosen]

Github user JoshRosen commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112567996
  
In fact, the old code did not require `JAVA_HOME` to be defined; it it 
wasn't, it would just fall back to `java`.  We should update the new Python 
code to preserve this behavior.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[JoshRosen]

Github user JoshRosen commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112567365
  
```python
[joshrosen Spark (4aecf09...)]$ ./dev/run-tests
Traceback (most recent call last):
  File "./dev/run-tests.py", line 536, in 
main()
  File "./dev/run-tests.py", line 479, in main
java_exe = determine_java_executable()
  File "./dev/run-tests.py", line 110, in determine_java_executable
java_exe = which(os.path.join(java_home, "bin", "java"))
  File "/Users/joshrosen/anaconda/lib/python2.7/posixpath.py", line 77, in 
join
elif path == '' or path.endswith('/'):
AttributeError: 'NoneType' object has no attribute 'endswith'
```

We could give a more helpful message when JAVA_HOME is not defined.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[AmplabJenkins]

Github user AmplabJenkins commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112546314
  
Merged build finished. Test PASSed.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[SparkQA]

Github user SparkQA commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112546082
  
  [Test build #34999 has 
finished](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/34999/console)
 for   PR 5694 at commit 
[`f9fbe54`](https://github.com/apache/spark/commit/f9fbe549165cca5c605dbb7d7b361891a407b7b1).
 * This patch **passes all tests**.
 * This patch merges cleanly.
 * This patch adds no public classes.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[AmplabJenkins]

Github user AmplabJenkins commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112516674
  
Merged build finished. Test FAILed.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[SparkQA]

Github user SparkQA commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112514711
  
  [Test build #34999 has 
started](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/34999/consoleFull)
 for   PR 5694 at commit 
[`f9fbe54`](https://github.com/apache/spark/commit/f9fbe549165cca5c605dbb7d7b361891a407b7b1).


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[AmplabJenkins]

Github user AmplabJenkins commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112514492
  
Merged build started.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[AmplabJenkins]

Github user AmplabJenkins commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112512688
  
 Merged build triggered.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[AmplabJenkins]

Github user AmplabJenkins commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112514474
  
 Merged build triggered.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[AmplabJenkins]

Github user AmplabJenkins commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112512715
  
Merged build started.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[brennonyork]

Github user brennonyork commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112509328
  
Sounds like a deal. I've got a separate thread with @shaneknapp on this one 
(he said the same thing re: the `jekyll` tool only on 
`amplab-jenkins-worker-01`) so understand on the revert here. Let me get that 
in place...


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[JoshRosen]

Github user JoshRosen commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112503112
  
```

Building Spark Documentation

[error] Cannot find a version of `jekyll` on the system; please install one 
and retry to build documentation.
```

AFAIK the doc builder dependencies are only installed on one of the nodes.  
For expediency's sake, I think we should just disable the documentation 
building in this PRB for now, but keep the logic which skips the full unit 
tests for doc-only changes.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[SparkQA]

Github user SparkQA commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112502119
  
  [Test build #34992 has 
finished](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/34992/console)
 for   PR 5694 at commit 
[`c42cf9a`](https://github.com/apache/spark/commit/c42cf9a8d75af7e3277ca3ddca0cb9dce857fa91).
 * This patch **passes all tests**.
 * This patch merges cleanly.
 * This patch adds no public classes.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[AmplabJenkins]

Github user AmplabJenkins commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112502389
  
Merged build finished. Test PASSed.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[AmplabJenkins]

Github user AmplabJenkins commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112494076
  
Merged build finished. Test FAILed.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[SparkQA]

Github user SparkQA commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112494070
  
  [Test build #34996 has 
finished](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/34996/console)
 for   PR 5694 at commit 
[`05d435b`](https://github.com/apache/spark/commit/05d435b6386e8ed4ac6f61c5a0713c157b5451d3).
 * This patch **fails to generate documentation**.
 * This patch merges cleanly.
 * This patch adds no public classes.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[SparkQA]

Github user SparkQA commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112493716
  
  [Test build #34996 has 
started](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/34996/consoleFull)
 for   PR 5694 at commit 
[`05d435b`](https://github.com/apache/spark/commit/05d435b6386e8ed4ac6f61c5a0713c157b5451d3).


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[AmplabJenkins]

Github user AmplabJenkins commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112493468
  
Merged build started.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[AmplabJenkins]

Github user AmplabJenkins commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112493428
  
 Merged build triggered.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[AmplabJenkins]

Github user AmplabJenkins commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112484480
  
Merged build finished. Test FAILed.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[SparkQA]

Github user SparkQA commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112484469
  
  [Test build #34995 has 
finished](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/34995/console)
 for   PR 5694 at commit 
[`2dff136`](https://github.com/apache/spark/commit/2dff136a6ed4c960e02af117add70e39da783037).
 * This patch **fails some tests**.
 * This patch merges cleanly.
 * This patch adds no public classes.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[SparkQA]

Github user SparkQA commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112483828
  
  [Test build #34995 has 
started](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/34995/consoleFull)
 for   PR 5694 at commit 
[`2dff136`](https://github.com/apache/spark/commit/2dff136a6ed4c960e02af117add70e39da783037).


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[AmplabJenkins]

Github user AmplabJenkins commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112483569
  
 Merged build triggered.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[AmplabJenkins]

Github user AmplabJenkins commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112483605
  
Merged build started.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[SparkQA]

Github user SparkQA commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112483403
  
  [Test build #34994 has 
finished](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/34994/console)
 for   PR 5694 at commit 
[`767a668`](https://github.com/apache/spark/commit/767a668c4b066af98b1f2ec6b2871c79f06289d8).
 * This patch **fails some tests**.
 * This patch merges cleanly.
 * This patch adds no public classes.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[AmplabJenkins]

Github user AmplabJenkins commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112483411
  
Merged build finished. Test FAILed.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[SparkQA]

Github user SparkQA commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112483009
  
  [Test build #34994 has 
started](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/34994/consoleFull)
 for   PR 5694 at commit 
[`767a668`](https://github.com/apache/spark/commit/767a668c4b066af98b1f2ec6b2871c79f06289d8).


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[brennonyork]

Github user brennonyork commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112482703
  
@JoshRosen just FYI i forgot to commit the code that would actually 
**build** the documentation yesterday (the `jekyll build` call) so retesting 
now, but if this passes (and builds docs) then I can revert the simple doc 
change and it should be ready!


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[AmplabJenkins]

Github user AmplabJenkins commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112482490
  
 Merged build triggered.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[AmplabJenkins]

Github user AmplabJenkins commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112482518
  
Merged build started.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[SparkQA]

Github user SparkQA commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112470562
  
  [Test build #34992 has 
started](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/34992/consoleFull)
 for   PR 5694 at commit 
[`c42cf9a`](https://github.com/apache/spark/commit/c42cf9a8d75af7e3277ca3ddca0cb9dce857fa91).


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[AmplabJenkins]

Github user AmplabJenkins commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112470361
  
 Merged build triggered.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[AmplabJenkins]

Github user AmplabJenkins commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112470389
  
Merged build started.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[JoshRosen]

Github user JoshRosen commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112469434
  
Jenkins, retest this please.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[SparkQA]

Github user SparkQA commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112227585
  
  [Test build #34959 has 
finished](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/34959/console)
 for   PR 5694 at commit 
[`c42cf9a`](https://github.com/apache/spark/commit/c42cf9a8d75af7e3277ca3ddca0cb9dce857fa91).
 * This patch **passes all tests**.
 * This patch merges cleanly.
 * This patch adds no public classes.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[AmplabJenkins]

Github user AmplabJenkins commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112227615
  
Merged build finished. Test PASSed.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[SparkQA]

Github user SparkQA commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112200990
  
  [Test build #34959 has 
started](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/34959/consoleFull)
 for   PR 5694 at commit 
[`c42cf9a`](https://github.com/apache/spark/commit/c42cf9a8d75af7e3277ca3ddca0cb9dce857fa91).


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[AmplabJenkins]

Github user AmplabJenkins commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112199689
  
 Merged build triggered.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[AmplabJenkins]

Github user AmplabJenkins commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112199726
  
Merged build started.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[SparkQA]

Github user SparkQA commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112171219
  
  [Test build #34957 has 
finished](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/34957/console)
 for   PR 5694 at commit 
[`fb85a41`](https://github.com/apache/spark/commit/fb85a41acf66c69d5fd12052560bc9e0d135e03a).
 * This patch **fails some tests**.
 * This patch merges cleanly.
 * This patch adds no public classes.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[AmplabJenkins]

Github user AmplabJenkins commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112171222
  
Merged build finished. Test FAILed.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[SparkQA]

Github user SparkQA commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112170845
  
  [Test build #34957 has 
started](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/34957/consoleFull)
 for   PR 5694 at commit 
[`fb85a41`](https://github.com/apache/spark/commit/fb85a41acf66c69d5fd12052560bc9e0d135e03a).


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[AmplabJenkins]

Github user AmplabJenkins commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112169875
  
Merged build started.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[AmplabJenkins]

Github user AmplabJenkins commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112169804
  
 Merged build triggered.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[SparkQA]

Github user SparkQA commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112167513
  
  [Test build #34956 has 
finished](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/34956/console)
 for   PR 5694 at commit 
[`0379833`](https://github.com/apache/spark/commit/03798339522832042e1d282c9c0aead0abca0d65).
 * This patch **fails some tests**.
 * This patch merges cleanly.
 * This patch adds no public classes.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[AmplabJenkins]

Github user AmplabJenkins commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112167521
  
Merged build finished. Test FAILed.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[SparkQA]

Github user SparkQA commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112166730
  
  [Test build #34956 has 
started](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/34956/consoleFull)
 for   PR 5694 at commit 
[`0379833`](https://github.com/apache/spark/commit/03798339522832042e1d282c9c0aead0abca0d65).


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[AmplabJenkins]

Github user AmplabJenkins commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112165147
  
Merged build started.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[AmplabJenkins]

Github user AmplabJenkins commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112165024
  
 Merged build triggered.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[AmplabJenkins]

Github user AmplabJenkins commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112164502
  
Merged build finished. Test FAILed.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[SparkQA]

Github user SparkQA commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112164490
  
  [Test build #34955 has 
finished](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/34955/console)
 for   PR 5694 at commit 
[`aa03d9e`](https://github.com/apache/spark/commit/aa03d9e1adb43d927cb3d00520936b90340427d9).
 * This patch **fails some tests**.
 * This patch merges cleanly.
 * This patch adds no public classes.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[SparkQA]

Github user SparkQA commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112163559
  
  [Test build #34955 has 
started](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/34955/consoleFull)
 for   PR 5694 at commit 
[`aa03d9e`](https://github.com/apache/spark/commit/aa03d9e1adb43d927cb3d00520936b90340427d9).


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[AmplabJenkins]

Github user AmplabJenkins commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112163059
  
Merged build started.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[AmplabJenkins]

Github user AmplabJenkins commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-112163033
  
 Merged build triggered.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[AmplabJenkins]

Github user AmplabJenkins commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-111796170
  
Merged build finished. Test PASSed.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[SparkQA]

Github user SparkQA commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-111796164
  
  [Test build #34874 has 
finished](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/34874/console)
 for   PR 5694 at commit 
[`ec1ae78`](https://github.com/apache/spark/commit/ec1ae789bef6a308db3cfd1b5d609c8cf02f19d9).
 * This patch **passes all tests**.
 * This patch merges cleanly.
 * This patch adds no public classes.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[SparkQA]

Github user SparkQA commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-111785726
  
  [Test build #34874 has 
started](https://amplab.cs.berkeley.edu/jenkins/job/SparkPullRequestBuilder/34874/consoleFull)
 for   PR 5694 at commit 
[`ec1ae78`](https://github.com/apache/spark/commit/ec1ae789bef6a308db3cfd1b5d609c8cf02f19d9).


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[AmplabJenkins]

Github user AmplabJenkins commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-111785407
  
 Merged build triggered.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[AmplabJenkins]

Github user AmplabJenkins commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-111785410
  
Merged build started.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[JoshRosen]

Github user JoshRosen commented on a diff in the pull request:

https://github.com/apache/spark/pull/5694#discussion_r32372531
  
--- Diff: dev/run-tests.py ---
@@ -0,0 +1,506 @@
+#!/usr/bin/env python2
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+import re
+import sys
+import shutil
+import subprocess
+from collections import namedtuple
+
+SPARK_HOME = os.path.join(os.path.dirname(os.path.realpath(__file__)), 
"..")
+USER_HOME = os.environ.get("HOME")
+
+
+def get_error_codes(err_code_file):
+"""Function to retrieve all block numbers from the `run-tests-codes.sh`
+file to maintain backwards compatibility with the `run-tests-jenkins`
+script"""
+
+with open(err_code_file, 'r') as f:
+err_codes = [e.split()[1].strip().split('=')
+ for e in f if e.startswith("readonly")]
+return dict(err_codes)
+
+
+ERROR_CODES = get_error_codes(os.path.join(SPARK_HOME, 
"dev/run-tests-codes.sh"))
+
+
+def exit_from_command_with_retcode(cmd, retcode):
+print "[error] running", cmd, "; received return code", retcode
+sys.exit(int(os.environ.get("CURRENT_BLOCK", 255)))
+
+
+def rm_r(path):
+"""Given an arbitrary path properly remove it with the correct python
+construct if it exists
+- from: http://stackoverflow.com/a/9559881""";
+
+if os.path.isdir(path):
+shutil.rmtree(path)
+elif os.path.exists(path):
+os.remove(path)
+
+
+def run_cmd(cmd):
+"""Given a command as a list of arguments will attempt to execute the
+command from the determined SPARK_HOME directory and, on failure, print
+an error message"""
+
+if not isinstance(cmd, list):
+cmd = cmd.split()
+try:
+subprocess.check_call(cmd)
+except subprocess.CalledProcessError as e:
+exit_from_command_with_retcode(e.cmd, e.returncode)
+
+
+def is_exe(path):
+"""Check if a given path is an executable file
+- from: http://stackoverflow.com/a/377028""";
+
+return os.path.isfile(path) and os.access(path, os.X_OK)
+
+
+def which(program):
+"""Find and return the given program by its absolute path or 'None'
+- from: http://stackoverflow.com/a/377028""";
+
+fpath, fname = os.path.split(program)
+
+if fpath:
+if is_exe(program):
+return program
+else:
+for path in os.environ.get("PATH").split(os.pathsep):
+path = path.strip('"')
+exe_file = os.path.join(path, program)
+if is_exe(exe_file):
+return exe_file
+return None
+
+
+def determine_java_executable():
+"""Will return the path of the java executable that will be used by 
Spark's
+tests or `None`"""
+
+# Any changes in the way that Spark's build detects java must be 
reflected
+# here. Currently the build looks for $JAVA_HOME/bin/java then falls 
back to
+# the `java` executable on the path
+
+java_home = os.environ.get("JAVA_HOME")
+
+# check if there is an executable at $JAVA_HOME/bin/java
+java_exe = which(os.path.join(java_home, "bin/java"))
+# if the java_exe wasn't set, check for a `java` version on the $PATH
+return java_exe if java_exe else which("java")
+
+
+JavaVersion = namedtuple('JavaVersion', ['major', 'minor', 'patch', 
'update'])
+
+
+def determine_java_version(java_exe):
+"""Given a valid java executable will return its version in named 
tuple format
+with accessors '.major', '.minor', '.patch', '.update'"""
+
+raw_output = subprocess.check_output([java_exe, "-version"],
+ stderr=subprocess.STDOUT)
+raw_version_str = raw_output.split('\n')[0]  # eg 'java version 
"1.8.0_25"'
+version_str = raw_version_st

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[JoshRosen]

Github user JoshRosen commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-111737166
  
Maybe we can skip this for now, but if I could add one additional test 
module / component I would add one for `docs/` so that documentation-only 
changes don't cause a full test.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[JoshRosen]

Github user JoshRosen commented on a diff in the pull request:

https://github.com/apache/spark/pull/5694#discussion_r32372475
  
--- Diff: dev/run-tests.py ---
@@ -0,0 +1,506 @@
+#!/usr/bin/env python2
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+import re
+import sys
+import shutil
+import subprocess
+from collections import namedtuple
+
+SPARK_HOME = os.path.join(os.path.dirname(os.path.realpath(__file__)), 
"..")
+USER_HOME = os.environ.get("HOME")
+
+
+def get_error_codes(err_code_file):
+"""Function to retrieve all block numbers from the `run-tests-codes.sh`
+file to maintain backwards compatibility with the `run-tests-jenkins`
+script"""
+
+with open(err_code_file, 'r') as f:
+err_codes = [e.split()[1].strip().split('=')
+ for e in f if e.startswith("readonly")]
+return dict(err_codes)
+
+
+ERROR_CODES = get_error_codes(os.path.join(SPARK_HOME, 
"dev/run-tests-codes.sh"))
+
+
+def exit_from_command_with_retcode(cmd, retcode):
+print "[error] running", cmd, "; received return code", retcode
+sys.exit(int(os.environ.get("CURRENT_BLOCK", 255)))
+
+
+def rm_r(path):
+"""Given an arbitrary path properly remove it with the correct python
+construct if it exists
+- from: http://stackoverflow.com/a/9559881""";
+
+if os.path.isdir(path):
+shutil.rmtree(path)
+elif os.path.exists(path):
+os.remove(path)
+
+
+def run_cmd(cmd):
+"""Given a command as a list of arguments will attempt to execute the
+command from the determined SPARK_HOME directory and, on failure, print
+an error message"""
+
+if not isinstance(cmd, list):
+cmd = cmd.split()
+try:
+subprocess.check_call(cmd)
+except subprocess.CalledProcessError as e:
+exit_from_command_with_retcode(e.cmd, e.returncode)
+
+
+def is_exe(path):
+"""Check if a given path is an executable file
+- from: http://stackoverflow.com/a/377028""";
+
+return os.path.isfile(path) and os.access(path, os.X_OK)
+
+
+def which(program):
+"""Find and return the given program by its absolute path or 'None'
+- from: http://stackoverflow.com/a/377028""";
+
+fpath, fname = os.path.split(program)
+
+if fpath:
+if is_exe(program):
+return program
+else:
+for path in os.environ.get("PATH").split(os.pathsep):
+path = path.strip('"')
+exe_file = os.path.join(path, program)
+if is_exe(exe_file):
+return exe_file
+return None
+
+
+def determine_java_executable():
+"""Will return the path of the java executable that will be used by 
Spark's
+tests or `None`"""
+
+# Any changes in the way that Spark's build detects java must be 
reflected
+# here. Currently the build looks for $JAVA_HOME/bin/java then falls 
back to
+# the `java` executable on the path
+
+java_home = os.environ.get("JAVA_HOME")
+
+# check if there is an executable at $JAVA_HOME/bin/java
+java_exe = which(os.path.join(java_home, "bin/java"))
+# if the java_exe wasn't set, check for a `java` version on the $PATH
+return java_exe if java_exe else which("java")
+
+
+JavaVersion = namedtuple('JavaVersion', ['major', 'minor', 'patch', 
'update'])
+
+
+def determine_java_version(java_exe):
+"""Given a valid java executable will return its version in named 
tuple format
+with accessors '.major', '.minor', '.patch', '.update'"""
+
+raw_output = subprocess.check_output([java_exe, "-version"],
+ stderr=subprocess.STDOUT)
+raw_version_str = raw_output.split('\n')[0]  # eg 'java version 
"1.8.0_25"'
+version_str = raw_version_st

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[JoshRosen]

Github user JoshRosen commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-111736875
  
After some reflection, I think that the suggestions outlined in [my last 
high-level 
comment](https://github.com/apache/spark/pull/5694#issuecomment-111625224) 
might take a while to implement since they're essentially suggesting a 
different high-level organization of some of the test-selection logic. Although 
I think in the long-term we may want a design closer to what I proposed there, 
for now I think we should set that aside and just focus on making the smaller 
changes that I suggested to get the current design working. If we do that, I'll 
commit this and will submit a followup PR to explore my reorganization ideas.

I'll go take a look at my previous comments now and will flag anything that 
doesn't need to be done as part of this PR.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[SparkQA]

Github user SparkQA commented on the pull request:

https://github.com/apache/spark/pull/5694#issuecomment-111646724
  
**[Test build #908 timed 
out](https://amplab.cs.berkeley.edu/jenkins/job/NewSparkPullRequestBuilder/908/console)**
 for PR 5694 at commit 
[`b7c72b9`](https://github.com/apache/spark/commit/b7c72b9cbae34c71478dca06f02184f6b317f58b)
 after a configured wait of `175m`.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---

-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[JoshRosen]

Github user JoshRosen commented on a diff in the pull request:

https://github.com/apache/spark/pull/5694#discussion_r32359387
  
--- Diff: dev/run-tests.py ---
@@ -0,0 +1,506 @@
+#!/usr/bin/env python2
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+import re
+import sys
+import shutil
+import subprocess
+from collections import namedtuple
+
+SPARK_HOME = os.path.join(os.path.dirname(os.path.realpath(__file__)), 
"..")
+USER_HOME = os.environ.get("HOME")
+
+
+def get_error_codes(err_code_file):
+"""Function to retrieve all block numbers from the `run-tests-codes.sh`
+file to maintain backwards compatibility with the `run-tests-jenkins`
+script"""
+
+with open(err_code_file, 'r') as f:
+err_codes = [e.split()[1].strip().split('=')
+ for e in f if e.startswith("readonly")]
+return dict(err_codes)
+
+
+ERROR_CODES = get_error_codes(os.path.join(SPARK_HOME, 
"dev/run-tests-codes.sh"))
+
+
+def exit_from_command_with_retcode(cmd, retcode):
+print "[error] running", cmd, "; received return code", retcode
+sys.exit(int(os.environ.get("CURRENT_BLOCK", 255)))
+
+
+def rm_r(path):
+"""Given an arbitrary path properly remove it with the correct python
+construct if it exists
+- from: http://stackoverflow.com/a/9559881""";
+
+if os.path.isdir(path):
+shutil.rmtree(path)
+elif os.path.exists(path):
+os.remove(path)
+
+
+def run_cmd(cmd):
+"""Given a command as a list of arguments will attempt to execute the
+command from the determined SPARK_HOME directory and, on failure, print
+an error message"""
+
+if not isinstance(cmd, list):
+cmd = cmd.split()
+try:
+subprocess.check_call(cmd)
+except subprocess.CalledProcessError as e:
+exit_from_command_with_retcode(e.cmd, e.returncode)
+
+
+def is_exe(path):
+"""Check if a given path is an executable file
+- from: http://stackoverflow.com/a/377028""";
+
+return os.path.isfile(path) and os.access(path, os.X_OK)
+
+
+def which(program):
+"""Find and return the given program by its absolute path or 'None'
+- from: http://stackoverflow.com/a/377028""";
+
+fpath, fname = os.path.split(program)
+
+if fpath:
+if is_exe(program):
+return program
+else:
+for path in os.environ.get("PATH").split(os.pathsep):
+path = path.strip('"')
+exe_file = os.path.join(path, program)
+if is_exe(exe_file):
+return exe_file
+return None
+
+
+def determine_java_executable():
+"""Will return the path of the java executable that will be used by 
Spark's
+tests or `None`"""
+
+# Any changes in the way that Spark's build detects java must be 
reflected
+# here. Currently the build looks for $JAVA_HOME/bin/java then falls 
back to
+# the `java` executable on the path
+
+java_home = os.environ.get("JAVA_HOME")
+
+# check if there is an executable at $JAVA_HOME/bin/java
+java_exe = which(os.path.join(java_home, "bin/java"))
+# if the java_exe wasn't set, check for a `java` version on the $PATH
+return java_exe if java_exe else which("java")
+
+
+JavaVersion = namedtuple('JavaVersion', ['major', 'minor', 'patch', 
'update'])
+
+
+def determine_java_version(java_exe):
+"""Given a valid java executable will return its version in named 
tuple format
+with accessors '.major', '.minor', '.patch', '.update'"""
+
+raw_output = subprocess.check_output([java_exe, "-version"],
+ stderr=subprocess.STDOUT)
+raw_version_str = raw_output.split('\n')[0]  # eg 'java version 
"1.8.0_25"'
+version_str = raw_version_st

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[JoshRosen]

Github user JoshRosen commented on a diff in the pull request:

https://github.com/apache/spark/pull/5694#discussion_r32359342
  
--- Diff: dev/run-tests.py ---
@@ -0,0 +1,506 @@
+#!/usr/bin/env python2
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+import re
+import sys
+import shutil
+import subprocess
+from collections import namedtuple
+
+SPARK_HOME = os.path.join(os.path.dirname(os.path.realpath(__file__)), 
"..")
+USER_HOME = os.environ.get("HOME")
+
+
+def get_error_codes(err_code_file):
+"""Function to retrieve all block numbers from the `run-tests-codes.sh`
+file to maintain backwards compatibility with the `run-tests-jenkins`
+script"""
+
+with open(err_code_file, 'r') as f:
+err_codes = [e.split()[1].strip().split('=')
+ for e in f if e.startswith("readonly")]
+return dict(err_codes)
+
+
+ERROR_CODES = get_error_codes(os.path.join(SPARK_HOME, 
"dev/run-tests-codes.sh"))
+
+
+def exit_from_command_with_retcode(cmd, retcode):
+print "[error] running", cmd, "; received return code", retcode
+sys.exit(int(os.environ.get("CURRENT_BLOCK", 255)))
+
+
+def rm_r(path):
+"""Given an arbitrary path properly remove it with the correct python
+construct if it exists
+- from: http://stackoverflow.com/a/9559881""";
+
+if os.path.isdir(path):
+shutil.rmtree(path)
+elif os.path.exists(path):
+os.remove(path)
+
+
+def run_cmd(cmd):
+"""Given a command as a list of arguments will attempt to execute the
+command from the determined SPARK_HOME directory and, on failure, print
+an error message"""
+
+if not isinstance(cmd, list):
+cmd = cmd.split()
+try:
+subprocess.check_call(cmd)
+except subprocess.CalledProcessError as e:
+exit_from_command_with_retcode(e.cmd, e.returncode)
+
+
+def is_exe(path):
+"""Check if a given path is an executable file
+- from: http://stackoverflow.com/a/377028""";
+
+return os.path.isfile(path) and os.access(path, os.X_OK)
+
+
+def which(program):
+"""Find and return the given program by its absolute path or 'None'
+- from: http://stackoverflow.com/a/377028""";
+
+fpath, fname = os.path.split(program)
+
+if fpath:
+if is_exe(program):
+return program
+else:
+for path in os.environ.get("PATH").split(os.pathsep):
+path = path.strip('"')
+exe_file = os.path.join(path, program)
+if is_exe(exe_file):
+return exe_file
+return None
+
+
+def determine_java_executable():
+"""Will return the path of the java executable that will be used by 
Spark's
+tests or `None`"""
+
+# Any changes in the way that Spark's build detects java must be 
reflected
+# here. Currently the build looks for $JAVA_HOME/bin/java then falls 
back to
+# the `java` executable on the path
+
+java_home = os.environ.get("JAVA_HOME")
+
+# check if there is an executable at $JAVA_HOME/bin/java
+java_exe = which(os.path.join(java_home, "bin/java"))
+# if the java_exe wasn't set, check for a `java` version on the $PATH
+return java_exe if java_exe else which("java")
+
+
+JavaVersion = namedtuple('JavaVersion', ['major', 'minor', 'patch', 
'update'])
+
+
+def determine_java_version(java_exe):
+"""Given a valid java executable will return its version in named 
tuple format
+with accessors '.major', '.minor', '.patch', '.update'"""
+
+raw_output = subprocess.check_output([java_exe, "-version"],
+ stderr=subprocess.STDOUT)
+raw_version_str = raw_output.split('\n')[0]  # eg 'java version 
"1.8.0_25"'
+version_str = raw_version_st

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[JoshRosen]

Github user JoshRosen commented on a diff in the pull request:

https://github.com/apache/spark/pull/5694#discussion_r32359302
  
--- Diff: dev/run-tests.py ---
@@ -0,0 +1,506 @@
+#!/usr/bin/env python2
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+import re
+import sys
+import shutil
+import subprocess
+from collections import namedtuple
+
+SPARK_HOME = os.path.join(os.path.dirname(os.path.realpath(__file__)), 
"..")
+USER_HOME = os.environ.get("HOME")
+
+
+def get_error_codes(err_code_file):
+"""Function to retrieve all block numbers from the `run-tests-codes.sh`
+file to maintain backwards compatibility with the `run-tests-jenkins`
+script"""
+
+with open(err_code_file, 'r') as f:
+err_codes = [e.split()[1].strip().split('=')
+ for e in f if e.startswith("readonly")]
+return dict(err_codes)
+
+
+ERROR_CODES = get_error_codes(os.path.join(SPARK_HOME, 
"dev/run-tests-codes.sh"))
+
+
+def exit_from_command_with_retcode(cmd, retcode):
+print "[error] running", cmd, "; received return code", retcode
+sys.exit(int(os.environ.get("CURRENT_BLOCK", 255)))
+
+
+def rm_r(path):
+"""Given an arbitrary path properly remove it with the correct python
+construct if it exists
+- from: http://stackoverflow.com/a/9559881""";
+
+if os.path.isdir(path):
+shutil.rmtree(path)
+elif os.path.exists(path):
+os.remove(path)
+
+
+def run_cmd(cmd):
+"""Given a command as a list of arguments will attempt to execute the
+command from the determined SPARK_HOME directory and, on failure, print
+an error message"""
+
+if not isinstance(cmd, list):
+cmd = cmd.split()
+try:
+subprocess.check_call(cmd)
+except subprocess.CalledProcessError as e:
+exit_from_command_with_retcode(e.cmd, e.returncode)
+
+
+def is_exe(path):
+"""Check if a given path is an executable file
+- from: http://stackoverflow.com/a/377028""";
+
+return os.path.isfile(path) and os.access(path, os.X_OK)
+
+
+def which(program):
+"""Find and return the given program by its absolute path or 'None'
+- from: http://stackoverflow.com/a/377028""";
+
+fpath, fname = os.path.split(program)
+
+if fpath:
+if is_exe(program):
+return program
+else:
+for path in os.environ.get("PATH").split(os.pathsep):
+path = path.strip('"')
+exe_file = os.path.join(path, program)
+if is_exe(exe_file):
+return exe_file
+return None
+
+
+def determine_java_executable():
+"""Will return the path of the java executable that will be used by 
Spark's
+tests or `None`"""
+
+# Any changes in the way that Spark's build detects java must be 
reflected
+# here. Currently the build looks for $JAVA_HOME/bin/java then falls 
back to
+# the `java` executable on the path
+
+java_home = os.environ.get("JAVA_HOME")
+
+# check if there is an executable at $JAVA_HOME/bin/java
+java_exe = which(os.path.join(java_home, "bin/java"))
+# if the java_exe wasn't set, check for a `java` version on the $PATH
+return java_exe if java_exe else which("java")
+
+
+JavaVersion = namedtuple('JavaVersion', ['major', 'minor', 'patch', 
'update'])
+
+
+def determine_java_version(java_exe):
+"""Given a valid java executable will return its version in named 
tuple format
+with accessors '.major', '.minor', '.patch', '.update'"""
+
+raw_output = subprocess.check_output([java_exe, "-version"],
+ stderr=subprocess.STDOUT)
+raw_version_str = raw_output.split('\n')[0]  # eg 'java version 
"1.8.0_25"'
+version_str = raw_version_st

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[JoshRosen]

Github user JoshRosen commented on a diff in the pull request:

https://github.com/apache/spark/pull/5694#discussion_r32359200
  
--- Diff: dev/run-tests.py ---
@@ -0,0 +1,506 @@
+#!/usr/bin/env python2
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+import re
+import sys
+import shutil
+import subprocess
+from collections import namedtuple
+
+SPARK_HOME = os.path.join(os.path.dirname(os.path.realpath(__file__)), 
"..")
+USER_HOME = os.environ.get("HOME")
+
+
+def get_error_codes(err_code_file):
+"""Function to retrieve all block numbers from the `run-tests-codes.sh`
+file to maintain backwards compatibility with the `run-tests-jenkins`
+script"""
+
+with open(err_code_file, 'r') as f:
+err_codes = [e.split()[1].strip().split('=')
+ for e in f if e.startswith("readonly")]
+return dict(err_codes)
+
+
+ERROR_CODES = get_error_codes(os.path.join(SPARK_HOME, 
"dev/run-tests-codes.sh"))
+
+
+def exit_from_command_with_retcode(cmd, retcode):
+print "[error] running", cmd, "; received return code", retcode
+sys.exit(int(os.environ.get("CURRENT_BLOCK", 255)))
+
+
+def rm_r(path):
+"""Given an arbitrary path properly remove it with the correct python
+construct if it exists
+- from: http://stackoverflow.com/a/9559881""";
+
+if os.path.isdir(path):
+shutil.rmtree(path)
+elif os.path.exists(path):
+os.remove(path)
+
+
+def run_cmd(cmd):
+"""Given a command as a list of arguments will attempt to execute the
+command from the determined SPARK_HOME directory and, on failure, print
+an error message"""
+
+if not isinstance(cmd, list):
+cmd = cmd.split()
+try:
+subprocess.check_call(cmd)
+except subprocess.CalledProcessError as e:
+exit_from_command_with_retcode(e.cmd, e.returncode)
+
+
+def is_exe(path):
+"""Check if a given path is an executable file
+- from: http://stackoverflow.com/a/377028""";
+
+return os.path.isfile(path) and os.access(path, os.X_OK)
+
+
+def which(program):
+"""Find and return the given program by its absolute path or 'None'
+- from: http://stackoverflow.com/a/377028""";
+
+fpath, fname = os.path.split(program)
+
+if fpath:
+if is_exe(program):
+return program
+else:
+for path in os.environ.get("PATH").split(os.pathsep):
+path = path.strip('"')
+exe_file = os.path.join(path, program)
+if is_exe(exe_file):
+return exe_file
+return None
+
+
+def determine_java_executable():
+"""Will return the path of the java executable that will be used by 
Spark's
+tests or `None`"""
+
+# Any changes in the way that Spark's build detects java must be 
reflected
+# here. Currently the build looks for $JAVA_HOME/bin/java then falls 
back to
+# the `java` executable on the path
+
+java_home = os.environ.get("JAVA_HOME")
+
+# check if there is an executable at $JAVA_HOME/bin/java
+java_exe = which(os.path.join(java_home, "bin/java"))
+# if the java_exe wasn't set, check for a `java` version on the $PATH
+return java_exe if java_exe else which("java")
+
+
+JavaVersion = namedtuple('JavaVersion', ['major', 'minor', 'patch', 
'update'])
+
+
+def determine_java_version(java_exe):
+"""Given a valid java executable will return its version in named 
tuple format
+with accessors '.major', '.minor', '.patch', '.update'"""
+
+raw_output = subprocess.check_output([java_exe, "-version"],
+ stderr=subprocess.STDOUT)
+raw_version_str = raw_output.split('\n')[0]  # eg 'java version 
"1.8.0_25"'
+version_str = raw_version_st

[GitHub] spark pull request: [SPARK-7017][Build][Project Infra]: Refactor d...

[JoshRosen]

Github user JoshRosen commented on a diff in the pull request:

https://github.com/apache/spark/pull/5694#discussion_r32359143
  
--- Diff: dev/run-tests.py ---
@@ -0,0 +1,506 @@
+#!/usr/bin/env python2
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+import re
+import sys
+import shutil
+import subprocess
+from collections import namedtuple
+
+SPARK_HOME = os.path.join(os.path.dirname(os.path.realpath(__file__)), 
"..")
+USER_HOME = os.environ.get("HOME")
+
+
+def get_error_codes(err_code_file):
+"""Function to retrieve all block numbers from the `run-tests-codes.sh`
+file to maintain backwards compatibility with the `run-tests-jenkins`
+script"""
+
+with open(err_code_file, 'r') as f:
+err_codes = [e.split()[1].strip().split('=')
+ for e in f if e.startswith("readonly")]
+return dict(err_codes)
+
+
+ERROR_CODES = get_error_codes(os.path.join(SPARK_HOME, 
"dev/run-tests-codes.sh"))
+
+
+def exit_from_command_with_retcode(cmd, retcode):
+print "[error] running", cmd, "; received return code", retcode
+sys.exit(int(os.environ.get("CURRENT_BLOCK", 255)))
+
+
+def rm_r(path):
+"""Given an arbitrary path properly remove it with the correct python
+construct if it exists
+- from: http://stackoverflow.com/a/9559881""";
+
+if os.path.isdir(path):
+shutil.rmtree(path)
+elif os.path.exists(path):
+os.remove(path)
+
+
+def run_cmd(cmd):
+"""Given a command as a list of arguments will attempt to execute the
+command from the determined SPARK_HOME directory and, on failure, print
+an error message"""
+
+if not isinstance(cmd, list):
+cmd = cmd.split()
+try:
+subprocess.check_call(cmd)
+except subprocess.CalledProcessError as e:
+exit_from_command_with_retcode(e.cmd, e.returncode)
+
+
+def is_exe(path):
+"""Check if a given path is an executable file
+- from: http://stackoverflow.com/a/377028""";
+
+return os.path.isfile(path) and os.access(path, os.X_OK)
+
+
+def which(program):
+"""Find and return the given program by its absolute path or 'None'
+- from: http://stackoverflow.com/a/377028""";
+
+fpath, fname = os.path.split(program)
+
+if fpath:
+if is_exe(program):
+return program
+else:
+for path in os.environ.get("PATH").split(os.pathsep):
+path = path.strip('"')
+exe_file = os.path.join(path, program)
+if is_exe(exe_file):
+return exe_file
+return None
+
+
+def determine_java_executable():
+"""Will return the path of the java executable that will be used by 
Spark's
+tests or `None`"""
+
+# Any changes in the way that Spark's build detects java must be 
reflected
+# here. Currently the build looks for $JAVA_HOME/bin/java then falls 
back to
+# the `java` executable on the path
+
+java_home = os.environ.get("JAVA_HOME")
+
+# check if there is an executable at $JAVA_HOME/bin/java
+java_exe = which(os.path.join(java_home, "bin/java"))
+# if the java_exe wasn't set, check for a `java` version on the $PATH
+return java_exe if java_exe else which("java")
+
+
+JavaVersion = namedtuple('JavaVersion', ['major', 'minor', 'patch', 
'update'])
+
+
+def determine_java_version(java_exe):
+"""Given a valid java executable will return its version in named 
tuple format
+with accessors '.major', '.minor', '.patch', '.update'"""
+
+raw_output = subprocess.check_output([java_exe, "-version"],
+ stderr=subprocess.STDOUT)
+raw_version_str = raw_output.split('\n')[0]  # eg 'java version 
"1.8.0_25"'
+version_str = raw_version_st