Repository: arrow Updated Branches: refs/heads/master 2288bfc18 -> b4eec6203
ARROW-932: [Python] Fix MSVC compiler warnings, build Python with /WX and -Werror in CI Author: Wes McKinney <[email protected]> Closes #913 from wesm/ARROW-932 and squashes the following commits: 9534ae9d [Wes McKinney] Only pass PYARROW_CXXFLAGS when set dedcbb9a [Wes McKinney] Fix typo b5a6d9af [Wes McKinney] Supress another clang warning 2e8f1053 [Wes McKinney] typo 5740f001 [Wes McKinney] Add PYARROW_CXXFLAGS option, fix MSVC compiler warnings c32ee092 [Wes McKinney] Remove print statement. Disable MSVC 4190 warning Project: http://git-wip-us.apache.org/repos/asf/arrow/repo Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/b4eec620 Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/b4eec620 Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/b4eec620 Branch: refs/heads/master Commit: b4eec62030c5bc54ff0161d97d8e85c7a963e592 Parents: 2288bfc Author: Wes McKinney <[email protected]> Authored: Sun Jul 30 12:58:16 2017 -0400 Committer: Wes McKinney <[email protected]> Committed: Sun Jul 30 12:58:16 2017 -0400 ---------------------------------------------------------------------- ci/msvc-build.bat | 3 +++ ci/travis_script_python.sh | 1 + python/CMakeLists.txt | 11 +++++++++++ python/doc/source/development.rst | 1 - python/pyarrow/_parquet.pyx | 2 +- python/pyarrow/includes/libarrow.pxd | 2 +- python/pyarrow/io.pxi | 2 +- python/pyarrow/scalar.pxi | 1 - python/pyarrow/table.pxi | 10 +++++----- python/pyarrow/types.pxi | 8 ++++---- python/setup.py | 6 ++++++ 11 files changed, 33 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/arrow/blob/b4eec620/ci/msvc-build.bat ---------------------------------------------------------------------- diff --git a/ci/msvc-build.bat b/ci/msvc-build.bat index 2a53776..04fe2ab 100644 --- a/ci/msvc-build.bat +++ b/ci/msvc-build.bat @@ -114,6 +114,9 @@ popd @rem see PARQUET-1018 pushd python + +set PYARROW_CXXFLAGS=/WX python setup.py build_ext --inplace --with-parquet --bundle-arrow-cpp bdist_wheel || exit /B py.test pyarrow -v -s --parquet || exit /B + popd http://git-wip-us.apache.org/repos/asf/arrow/blob/b4eec620/ci/travis_script_python.sh ---------------------------------------------------------------------- diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh index 907bc60..9135aaf 100755 --- a/ci/travis_script_python.sh +++ b/ci/travis_script_python.sh @@ -23,6 +23,7 @@ source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh export ARROW_HOME=$ARROW_CPP_INSTALL export PARQUET_HOME=$TRAVIS_BUILD_DIR/parquet-env export LD_LIBRARY_PATH=$ARROW_HOME/lib:$PARQUET_HOME/lib:$LD_LIBRARY_PATH +export PYARROW_CXXFLAGS="-Werror" build_parquet_cpp() { export PARQUET_ARROW_VERSION=$(git rev-parse HEAD) http://git-wip-us.apache.org/repos/asf/arrow/blob/b4eec620/python/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 846e4dd..bfae157 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -57,6 +57,8 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") option(PYARROW_BUNDLE_ARROW_CPP "Bundle the Arrow C++ libraries" OFF) + set(PYARROW_CXXFLAGS "" CACHE STRING + "Compiler flags to append when compiling Arrow") endif() find_program(CCACHE_FOUND ccache) @@ -75,6 +77,7 @@ include(CompilerInfo) # Add common flags set(CMAKE_CXX_FLAGS "${CXX_COMMON_FLAGS} ${CMAKE_CXX_FLAGS}") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${PYARROW_CXXFLAGS}") if (NOT MSVC) # Enable perf and other tools to work properly @@ -82,6 +85,13 @@ if (NOT MSVC) # Suppress Cython warnings set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-variable") +else() + # MSVC version of -Wno-return-type-c-linkage + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4190") + + # Cython generates some bitshift expressions that MSVC does not like in + # __Pyx_PyFloat_DivideObjC + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4293") endif() if ("${COMPILER_FAMILY}" STREQUAL "clang") @@ -95,6 +105,7 @@ if ("${COMPILER_FAMILY}" STREQUAL "clang") # Cython warnings in clang set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-parentheses-equality") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-constant-logical-operand") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-missing-declarations") # We have public Cython APIs which return C++ types, which are in an extern # "C" blog (no symbol mangling) and clang doesn't like this http://git-wip-us.apache.org/repos/asf/arrow/blob/b4eec620/python/doc/source/development.rst ---------------------------------------------------------------------- diff --git a/python/doc/source/development.rst b/python/doc/source/development.rst index 55b3efd..d0a1c54 100644 --- a/python/doc/source/development.rst +++ b/python/doc/source/development.rst @@ -267,7 +267,6 @@ Now, we build and install Arrow C++ libraries -DCMAKE_INSTALL_PREFIX=%ARROW_HOME% ^ -DCMAKE_BUILD_TYPE=Release ^ -DARROW_BUILD_TESTS=off ^ - -DARROW_ZLIB_VENDORED=off ^ -DARROW_PYTHON=on .. cmake --build . --target INSTALL --config Release cd ..\.. http://git-wip-us.apache.org/repos/asf/arrow/blob/b4eec620/python/pyarrow/_parquet.pyx ---------------------------------------------------------------------- diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx index aef6618..c940122 100644 --- a/python/pyarrow/_parquet.pyx +++ b/python/pyarrow/_parquet.pyx @@ -636,7 +636,7 @@ cdef class ParquetWriter: elif row_group_size == 0: raise ValueError('Row group size cannot be 0') - cdef int c_row_group_size = row_group_size + cdef int64_t c_row_group_size = row_group_size with nogil: check_status(self.writer.get() http://git-wip-us.apache.org/repos/asf/arrow/blob/b4eec620/python/pyarrow/includes/libarrow.pxd ---------------------------------------------------------------------- diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd index ffe867b..db6770f 100644 --- a/python/pyarrow/includes/libarrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -369,7 +369,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil: shared_ptr[CTable]* table) int num_columns() - int num_rows() + int64_t num_rows() c_bool Equals(const CTable& other) http://git-wip-us.apache.org/repos/asf/arrow/blob/b4eec620/python/pyarrow/io.pxi ---------------------------------------------------------------------- diff --git a/python/pyarrow/io.pxi b/python/pyarrow/io.pxi index 9e4e907..01c987d 100644 --- a/python/pyarrow/io.pxi +++ b/python/pyarrow/io.pxi @@ -923,7 +923,7 @@ cdef class _HdfsClient: cdef c_string c_path = tobytes(path) with nogil: check_status(self.client.get() - .Delete(c_path, recursive)) + .Delete(c_path, recursive == 1)) def open(self, path, mode='rb', buffer_size=None, replication=None, default_block_size=None): http://git-wip-us.apache.org/repos/asf/arrow/blob/b4eec620/python/pyarrow/scalar.pxi ---------------------------------------------------------------------- diff --git a/python/pyarrow/scalar.pxi b/python/pyarrow/scalar.pxi index dec5341..1f72070 100644 --- a/python/pyarrow/scalar.pxi +++ b/python/pyarrow/scalar.pxi @@ -169,7 +169,6 @@ cdef class Time64Value(ArrayValue): CTime64Type* dtype = <CTime64Type*> ap.type().get() cdef int64_t val = ap.Value(self.index) - print(val) if dtype.unit() == TimeUnit_MICRO: return (datetime.datetime(1970, 1, 1) + datetime.timedelta(microseconds=val)).time() http://git-wip-us.apache.org/repos/asf/arrow/blob/b4eec620/python/pyarrow/table.pxi ---------------------------------------------------------------------- diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index c1d5a50..6277761 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -286,7 +286,7 @@ cdef int _schema_from_arrays( c_string c_name vector[shared_ptr[CField]] fields shared_ptr[CDataType] type_ - int K = len(arrays) + Py_ssize_t K = len(arrays) fields.resize(K) @@ -733,7 +733,7 @@ cdef class Table: vector[shared_ptr[CColumn]] columns shared_ptr[CSchema] schema shared_ptr[CTable] table - size_t K = len(arrays) + int i, K = <int> len(arrays) _schema_from_arrays(arrays, names, metadata, &schema) @@ -841,7 +841,7 @@ cdef class Table: self._check_nullptr() return pyarrow_wrap_schema(self.table.schema()) - def column(self, int64_t i): + def column(self, int i): """ Select a column by its numeric index. @@ -855,8 +855,8 @@ cdef class Table: """ cdef: Column column = Column() - int64_t num_columns = self.num_columns - int64_t index + int num_columns = self.num_columns + int index self._check_nullptr() if not -num_columns <= i < num_columns: http://git-wip-us.apache.org/repos/asf/arrow/blob/b4eec620/python/pyarrow/types.pxi ---------------------------------------------------------------------- diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi index a8d7aa0..fefde55 100644 --- a/python/pyarrow/types.pxi +++ b/python/pyarrow/types.pxi @@ -281,12 +281,12 @@ cdef class Schema: def __len__(self): return self.schema.num_fields() - def __getitem__(self, int64_t i): + def __getitem__(self, int i): cdef: Field result = Field() - int64_t num_fields = self.schema.num_fields() - int64_t index + int num_fields = self.schema.num_fields() + int index if not -num_fields <= i < num_fields: raise IndexError( @@ -456,7 +456,7 @@ def field(name, DataType type, bint nullable=True, dict metadata=None): convert_metadata(metadata, &c_meta) result.sp_field.reset(new CField(tobytes(name), type.sp_type, - nullable, c_meta)) + nullable == 1, c_meta)) result.field = result.sp_field.get() result.type = type return result http://git-wip-us.apache.org/repos/asf/arrow/blob/b4eec620/python/setup.py ---------------------------------------------------------------------- diff --git a/python/setup.py b/python/setup.py index 801cd17..ebf28cc 100644 --- a/python/setup.py +++ b/python/setup.py @@ -92,6 +92,8 @@ class build_ext(_build_ext): self.extra_cmake_args = os.environ.get('PYARROW_CMAKE_OPTIONS', '') self.build_type = os.environ.get('PYARROW_BUILD_TYPE', 'debug').lower() + self.cmake_cxxflags = os.environ.get('PYARROW_CXXFLAGS', '') + if sys.platform == 'win32': # Cannot do debug builds in Windows unless Python itself is a debug # build @@ -146,6 +148,10 @@ class build_ext(_build_ext): if self.with_plasma: cmake_options.append('-DPYARROW_BUILD_PLASMA=on') + if len(self.cmake_cxxflags) > 0: + cmake_options.append('-DPYARROW_CXXFLAGS="{0}"' + .format(self.cmake_cxxflags)) + if self.bundle_arrow_cpp: cmake_options.append('-DPYARROW_BUNDLE_ARROW_CPP=ON') # ARROW-1090: work around CMake rough edges
