This is an automated email from the ASF dual-hosted git repository. cjolivier01 pushed a commit to branch cython in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
commit 2fee93e23e58733d7022d9afe153eafe8438af59 Author: Olivier <cooli...@amazon.com> AuthorDate: Thu Mar 8 08:46:46 2018 -0800 Refreshed branch cython --- CMakeLists.txt | 67 +++- cmake/CythonUtil.cmake | 297 ++++++++++++++++ cmake/Modules/FindCython.cmake | 69 ++++ cmake/Modules/FindOpenBLAS.cmake | 9 +- cmake/UseCython.cmake | 390 +++++++++++++++++++++ python/mxnet/{_cy3 => cython}/__init__.py | 2 - python/mxnet/cython/base.pyi | 2 +- python/mxnet/{_cy3 => cython/cy2}/README | 0 python/mxnet/{_cy2 => cython/cy2}/__init__.py | 0 python/mxnet/{_cy2 => cython/cy3}/README | 0 python/mxnet/{_cy3 => cython/cy3}/__init__.py | 0 python/mxnet/cython/mxcython.pyx | 86 +++++ python/mxnet/{_cy2 => ndarray/cy2}/__init__.py | 0 python/mxnet/{_cy3 => ndarray/cy3}/__init__.py | 0 python/mxnet/{cython => ndarray}/ndarray.pyx | 4 +- python/mxnet/{_cy2 => symbol/cy2}/__init__.py | 0 python/mxnet/{_cy3 => symbol/cy3}/__init__.py | 0 python/mxnet/{cython => symbol}/symbol.pyx | 8 +- python/setup.py | 5 +- .../__init__.py => src/cython/cy2/CMakeLists.txt | 17 +- .../__init__.py => src/cython/cy3/CMakeLists.txt | 17 +- src/cython/cython_util.cc | 103 ++++++ src/cython/cython_util.h | 51 +++ tests/python/unittest/test_cython.py | 75 ++++ .../__init__.py => tools/cython/clean_cython.sh | 6 +- .../mxnet/_cy3/__init__.py => tools/cython/cydb2 | 11 +- .../mxnet/_cy3/__init__.py => tools/cython/cydb3 | 11 +- 27 files changed, 1198 insertions(+), 32 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b3a8955..04f0795 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + cmake_minimum_required(VERSION 3.0.2) project(mxnet C CXX) @@ -36,6 +53,7 @@ mxnet_option(ENABLE_CUDA_RTC "Build with CUDA runtime compilation support" mxnet_option(BUILD_CPP_EXAMPLES "Build cpp examples" ON) mxnet_option(INSTALL_EXAMPLES "Install the example source files." OFF) mxnet_option(USE_SIGNAL_HANDLER "Print stack traces on segfaults." OFF) +mxnet_option(INSTALL_CYTHON_INPLACE "Install cython modules into source python tree." ON) if(USE_CUDA AND NOT USE_OLDCMAKECUDA) message(STATUS "CMake version '${CMAKE_VERSION}' using generator '${CMAKE_GENERATOR}'") @@ -539,14 +557,14 @@ if(USE_CUDA) add_definitions(-DMXNET_USE_CUDA=1) if(CUDA_LIBRARY_PATH) if(IS_CONTAINER_BUILD) - # In case of building on a production-like build container which may not have Cuda installed - if(NOT CMAKE_SYSTEM_HAS_CUDA) - # Assuming building in a container that doesn't have CUDA installed (ie CPU-only build machine) - # so use the stub cuda driver shared library - if(EXISTS ${CUDA_LIBRARY_PATH}/stubs/libcuda.so) - link_directories(${CUDA_LIBRARY_PATH}/stubs) - endif() - endif() + # In case of building on a production-like build container which may not have Cuda installed + if(NOT CMAKE_SYSTEM_HAS_CUDA) + # Assuming building in a container that doesn't have CUDA installed (ie CPU-only build machine) + # so use the stub cuda driver shared library + if(EXISTS ${CUDA_LIBRARY_PATH}/stubs/libcuda.so) + link_directories(${CUDA_LIBRARY_PATH}/stubs) + endif() + endif() endif() endif() endif() @@ -574,6 +592,9 @@ else() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc") endif() +# +# MXNet libraries +# set(MXNET_INSTALL_TARGETS mxnet) if(UNIX) list(APPEND MXNET_INSTALL_TARGETS mxnet_static) @@ -586,12 +607,11 @@ else() add_library(mxnet SHARED ${SOURCE}) endif() -if(USE_CUDA) - if(FIRST_CUDA AND MSVC) - target_compile_options(mxnet PUBLIC "$<$<CONFIG:DEBUG>:-Xcompiler=-MTd>") - target_compile_options(mxnet PUBLIC "$<$<CONFIG:RELEASE>:-Xcompiler=-MT>") - endif() +if(USE_CUDA AND FIRST_CUDA AND MSVC) + target_compile_options(mxnet PUBLIC "$<$<CONFIG:DEBUG>:-Xcompiler=-MTd>") + target_compile_options(mxnet PUBLIC "$<$<CONFIG:RELEASE>:-Xcompiler=-MT>") endif() + if(USE_DIST_KVSTORE) if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/ps-lite/CMakeLists.txt) add_subdirectory("ps-lite") @@ -621,6 +641,27 @@ if(USE_PLUGINS_WARPCTC) target_link_libraries(mxnet PUBLIC optimized ${WARPCTC_LIB_RELEASE}) endif() +# +# BEGIN Cython Build +# +include(cmake/CythonUtil.cmake) +mxnet_external_build_cython(2 target_2) +mxnet_external_build_cython(3 target_3) + +if(INSTALL_CYTHON_INPLACE) + set(cython_install_targets mxnet ${target_2} ${target_3}) + cython_install_into_source_dir( + ${CMAKE_CURRENT_BINARY_DIR}/python + ${CMAKE_CURRENT_SOURCE_DIR}/python + cython_install_targets + ) +endif() + +add_custom_target(mxnet_runtime DEPENDS ${cython_install_targets}) +# +# END Cython build +# + if(USE_OPENCV) add_executable(im2rec "tools/im2rec.cc") diff --git a/cmake/CythonUtil.cmake b/cmake/CythonUtil.cmake new file mode 100644 index 0000000..f98bf61 --- /dev/null +++ b/cmake/CythonUtil.cmake @@ -0,0 +1,297 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +################################################################################################ +# Build a cython module +# +# Usage: +# mxnet_external_build_cython(<python major version>) +# +function(mxnet_build_cython_module python_version _cython_modules) + string(REGEX REPLACE "@" ";" PROP_MXNET_INCLUDE_DIRECTORIES "${MXNET_INCLUDE_DIRECTORIES}") + string(REGEX REPLACE "@" ";" PROP_MXNET_INTERFACE_LINK_LIBRARIES "${MXNET_INTERFACE_LINK_LIBRARIES}") + + foreach(var ${PROP_MXNET_INCLUDE_DIRECTORIES}) + include_directories(${var}) + endforeach() + + unset(PYTHONLIBS_FOUND) + unset(PYTHON_LIBRARIES) + unset(PYTHON_INCLUDE_PATH) + unset(PYTHON_INCLUDE_DIRS) + unset(PYTHON_DEBUG_LIBRARIES) + unset(PYTHONLIBS_VERSION_STRING) + unset(PYTHONINTERP_FOUND) + unset(PYTHON_EXECUTABLE) + unset(PYTHON_VERSION_STRING) + unset(PYTHON_VERSION_MAJOR) + unset(PYTHON_VERSION_MINOR) + unset(PYTHON_VERSION_PATCH) + + if(python_version EQUAL 2) + set(Python_ADDITIONAL_VERSIONS 2.7 2.6 2.5 2.4 2.3 2.2 2.1 2.0) + elseif(python_version EQUAL 3) + set(Python_ADDITIONAL_VERSIONS 3.7 3.6 3.5 3.4 3.3 3.2 3.1 3.0) + else() + message(FATAL_ERROR "Nov alid python_version set (must be 2 or 3)") + return() + endif() + + set(python_libs_version ${python_version}) + include(${MXNET_ROOT_DIR}/cmake/UseCython.cmake) # set from mxnet_external_build_cython + + if(NOT CYTHON${python_version}_FOUND) + message(WARNING " Could not build cython target for Python ${python_version}") + return() + endif() + + set(CYTHON_SUBDIR ".") + + file(GLOB_RECURSE CYTHON_SOURCE "${MXNET_ROOT_DIR}/python/mxnet/*.pyx") + + if(NOT MXNET_LIB_LOCATION) + set(MXNET_LIB_LOCATION mxnet) + endif() + + set(CYTHON_CXX_SOURCE "") + foreach(cy_file ${CYTHON_SOURCE}) + set_source_files_properties(${cy_file} PROPERTIES CYTHON_IS_CXX TRUE) + list(APPEND CYTHON_CXX_SOURCE ${cy_file_generated}) + get_filename_component(cy_module ${cy_file} NAME_WE) + get_filename_component(cy_dir ${cy_file} DIRECTORY) +# message(STATUS "MXNET_ROOT_DIR: ${MXNET_ROOT_DIR}") + + file(RELATIVE_PATH cy_directory "${MXNET_ROOT_DIR}" ${cy_dir}) +# message(STATUS "cy_directory: ${cy_directory}") +# message(STATUS "MXNET_BINARY_DIR: ${MXNET_BINARY_DIR}") + set(bin_directory "${MXNET_BINARY_DIR}/${cy_directory}") +# message(STATUS "bin_directory: ${bin_directory}") + set(CYTHON_SUBDIR "${bin_directory}/cy${python_version}") + #message(STATUS "CYTHON_SUBDIR: ${CYTHON_SUBDIR}") + file(MAKE_DIRECTORY ${CYTHON_SUBDIR}) + + # We need cmake to have different target names for python 2 and 3 + set(cy_module_name ${cy_module}) + + # cython_add_module expects cxx outyput dir to be relative to current binary dir + file(RELATIVE_PATH CYTHON_SUBDIR "${CMAKE_CURRENT_BINARY_DIR}" ${CYTHON_SUBDIR}) + #${CMAKE_CURRENT_BINARY_DIR}/${c_cxx_output_subdir} + + cython_add_module(${cy_module_name} + "${CYTHON_SUBDIR}" + "${MXNET_BINARY_DIR}/cython/cy${python_version}" + ${cy_file}) + + set_target_properties(${cy_module_name} + PROPERTIES + LIBRARY_OUTPUT_DIRECTORY "${CYTHON_SUBDIR}/" + INTERFACE_LINK_LIBRARIES "${PROP_MXNET_INTERFACE_LINK_LIBRARIES}" + ) + target_link_libraries(${cy_module_name} ${MXNET_LIB_LOCATION}) + list(APPEND output_cython_modules ${cy_module_name}) + endforeach() + set(${_cython_modules} ${output_cython_modules} PARENT_SCOPE) + #message(STATUS "output_cython_modules: ${output_cython_modules}") +endfunction() + +################################################################################################ +# Copy cython modules into source python dir +#function(_cython_install_into_source_dir python_version _cython_binary_dir _source_root_dir) +# file(GLOB_RECURSE cython_module_runtimes +# "${_cython_binary_dir}/*.so" +# "${_cython_binary_dir}/*.dll" +# "${_cython_binary_dir}/*.pyd") +# #message(STATUS "cython_module_runtimes: ${cython_module_runtimes}") +# set(_running_target_suffix "") +# foreach(_file ${cython_module_runtimes}) +# set(_running_target_suffix "${_running_target_suffix}_") +# message(STATUS "_file: ${_file}") +# get_filename_component(_cy_module_name ${_file} NAME_WE) +# get_filename_component(_cy_module_directory ${_file} DIRECTORY) +# #message(STATUS "_cy_module_name: ${_cy_module_name}") +# #message(STATUS "_cy_module_directory: ${_cy_module_directory}") +# file(RELATIVE_PATH _relpath_source ${_cython_binary_dir} ${_file}) +# #message(STATUS "_relpath_source: ${_relpath_source}") +# set(_dest_file "${_source_root_dir}/${_relpath_source}") +# #message(STATUS "_dest_file: ${_dest_file}") +# get_filename_component(_dest_file_dir ${_dest_file} DIRECTORY) +# file(MAKE_DIRECTORY ${_dest_file_dir}) +# #message(STATUS "${_file} -> ${_dest_file}") +# set(_target_name cython_${_cy_module_name}_copy_shared_obj) +# #message(STATUS "target: ${_target_name}") +# add_custom_target(${_target_name} ALL +# DEPENDS ${_cy_module_name} +# COMMAND ${CMAKE_COMMAND} -E copy_if_different ${_file} ${_dest_file}) +# add_custom_target(cython_${_cy_module_name}_copy_cython_debug ALL +# DEPENDS ${_cy_module_name} +# COMMAND ${CMAKE_COMMAND} -E copy_directory +# ${_cy_module_directory}/cython_debug +# ${_dest_file_dir}/cython_debug +# ) +# endforeach() +#endfunction() + +function(cython_install_into_source_dir + _cython_binary_dir + _source_root_dir + _dependencies + ) + file(GLOB_RECURSE cython_module_runtimes + "${_cython_binary_dir}/*.so" + "${_cython_binary_dir}/*.dll" + "${_cython_binary_dir}/*.pyd") + #message(STATUS "cython_module_runtimes: ${cython_module_runtimes}") + set(_running_target_suffix "") + foreach(_file ${cython_module_runtimes}) + #set(_running_target_suffix "${_running_target_suffix}_") + #message(STATUS "_file: ${_file}") + get_filename_component(_cy_module_name ${_file} NAME_WE) + get_filename_component(_cy_module_directory ${_file} DIRECTORY) + #message(STATUS "_cy_module_name: ${_cy_module_name}") + #message(STATUS "_cy_module_directory: ${_cy_module_directory}") + file(RELATIVE_PATH _relpath_source ${_cython_binary_dir} ${_file}) + #message(STATUS "_relpath_source: ${_relpath_source}") + set(_dest_file "${_source_root_dir}/${_relpath_source}") + #message(STATUS "_dest_file: ${_dest_file}") + get_filename_component(_dest_file_dir ${_dest_file} DIRECTORY) + file(MAKE_DIRECTORY ${_dest_file_dir}) + #message(STATUS "${_file} -> ${_dest_file}") + #message(STATUS "target: ${_target_name}") + #message(STATUS "_dependencies: ${${_dependencies}}") + file(RELATIVE_PATH _full_module_name ${CMAKE_CURRENT_SOURCE_DIR} ${_dest_file}) + string(REGEX REPLACE "/" "." _full_module_name "${_full_module_name}") + string(REGEX REPLACE "\\\\" "." _full_module_name "${_full_module_name}") + #message(STATUS "_full_module_name: ${_full_module_name}") +if(UNIX) + add_custom_target( + ${_full_module_name} ALL + DEPENDS ${${_dependencies}} + COMMAND ln -sf ${_file} ${_dest_file}) +else() + add_custom_target( + ${_full_module_name} ALL + DEPENDS ${${_dependencies}} + COMMAND ${CMAKE_COMMAND} -E copy ${_file} ${_dest_file}) +endif() + endforeach() +endfunction() + +################################################################################################ +# Spawn external CMakeLists.txt in order to build a particular cython/python version +# +# The spawn approach is because we need to detect and build with both python version 2 and 3 +# This is not osmething that a single process of cmake can deal with, so we launch +# a cmake process for the cython build 2, then for 3, passing it our relevant config +# +# Usage: +# mxnet_external_build_cython(<python major version>) +# +function(mxnet_external_build_cython python_major_version target) + set(PMV ${python_major_version}) + + if(CYTHON_WITHOUT_MXNET_TARGET) + set(CYTHON_DEPS "") + set(CYTHON_MXNET_LIB_LOCATION "") + else() + set(CYTHON_DEPS mxnet) + set(CYTHON_MXNET_LIB_LOCATION $<TARGET_LINKER_FILE:mxnet>) + endif() + + file(GLOB_RECURSE CYTHON_SOURCE "python/mxnet/cython/*.pyx") + + get_cmake_property(CACHE_VARS CACHE_VARIABLES) + foreach(_cache_var ${CACHE_VARS}) + #message(STATUS "${_cache_var}=${${_cache_var}}") + get_property(CACHE_VAR_HELPSTRING CACHE ${_cache_var} PROPERTY HELPSTRING) + if(NOT _cache_var MATCHES "CMAKE_EXTRA_GENERATOR_.+" + AND NOT _cache_var MATCHES "FIND_PACKAGE_MESSAGE_.+" + ) + if(_cache_var MATCHES "USE_.+" + OR _cache_var MATCHES "CMAKE_MODULE_.+" + OR CACHE_VAR_HELPSTRING STREQUAL "No help, variable specified on the command line." + ) + get_property(_cache_var_type CACHE ${_cache_var} PROPERTY TYPE) + if(_cache_var_type STREQUAL "UNINITIALIZED") + set(_cache_var_type) + else() + set(_cache_var_type :${_cache_var_type}) + endif() + set(CMAKE_ARGS "${CMAKE_ARGS} -D${_cache_var}${_cache_var_type}=\"${${_cache_var}}\"") + endif() + endif() + endforeach() + + get_property(PROP_MXNET_INCLUDE_DIRECTORIES TARGET mxnet PROPERTY INCLUDE_DIRECTORIES) + string(REGEX REPLACE "\;" "@" MXNET_INCLUDE_DIRECTORIES "${PROP_MXNET_INCLUDE_DIRECTORIES}") + + get_property(PROP_MXNET_INTERFACE_LINK_LIBRARIES TARGET mxnet PROPERTY INTERFACE_LINK_LIBRARIES) + string(REGEX REPLACE "\;" "@" MXNET_INTERFACE_LINK_LIBRARIES "${PROP_MXNET_INTERFACE_LINK_LIBRARIES}") + + set(CYTHON_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/cython/cy${PMV}) + + set(_config_cleanup_files "") + set(_config_cleanup_files_ex "") + + foreach(_file ${CYTHON_SOURCE}) + get_filename_component(_cy_module ${_file} NAME_WE) + list(APPEND _config_cleanup_files "${CYTHON_BINARY_DIR}/${_cy_module}.so") + list(APPEND _config_cleanup_files "${CYTHON_BINARY_DIR}/${_cy_module}.cxx") + list(APPEND _config_cleanup_files "${CYTHON_BINARY_DIR}/${_cy_module}.c") + endforeach() + + # Clear some cmake-generated files + list(APPEND _config_cleanup_files_ex "${CYTHON_BINARY_DIR}/CMakeCache.txt") + list(APPEND _config_cleanup_files_ex "${CYTHON_BINARY_DIR}/Makefile") + + # Get current cleanup files + get_directory_property(CLEANUP_FILES ADDITIONAL_MAKE_CLEAN_FILES) + list(APPEND CLEANUP_FILES ${_config_cleanup_files}) + list(APPEND CLEANUP_FILES ${_config_cleanup_files_ex}) + # Set new list of cleanup files + set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES "${CLEANUP_FILES}") + + file(MAKE_DIRECTORY ${CYTHON_BINARY_DIR}) + + add_custom_target(${PROJECT_NAME}_ConfigCython${PMV} ALL + ${CMAKE_COMMAND} + ${CMAKE_ARGS} + -G "${CMAKE_GENERATOR}" + -DCMAKE_MODULE_PATH="${CMAKE_MODULE_PATH}" + -DMXNET_INCLUDE_DIRECTORIES="${MXNET_INCLUDE_DIRECTORIES}" + -DMXNET_INTERFACE_LINK_LIBRARIES="${MXNET_INTERFACE_LINK_LIBRARIES}" + -DMXNET_LIB_LOCATION=${CYTHON_MXNET_LIB_LOCATION} + -DMXNET_ROOT_DIR=${CMAKE_CURRENT_SOURCE_DIR} + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS} + -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} + #-DINSTALL_CYTHON_INPLACE=${INSTALL_CYTHON_INPLACE} + -DMXNET_BINARY_DIR=${CMAKE_CURRENT_BINARY_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/src/cython/cy${PMV} + WORKING_DIRECTORY ${CYTHON_BINARY_DIR} + DEPENDS ${CYTHON_DEPS} + ) + + add_custom_target(${PROJECT_NAME}_BuildCython${PMV} ALL + ${CMAKE_COMMAND} + --build ${CYTHON_BINARY_DIR} + --config ${CMAKE_BUILD_TYPE} + WORKING_DIRECTORY ${CYTHON_BINARY_DIR} + DEPENDS ${PROJECT_NAME}_ConfigCython${PMV} + ) + + set(${target} ${PROJECT_NAME}_BuildCython${PMV} PARENT_SCOPE) +endfunction() diff --git a/cmake/Modules/FindCython.cmake b/cmake/Modules/FindCython.cmake new file mode 100644 index 0000000..132de50 --- /dev/null +++ b/cmake/Modules/FindCython.cmake @@ -0,0 +1,69 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Find the Cython compiler. +# +# This code sets the following variables: +# +# CYTHON_EXECUTABLE +# +# See also UseCython.cmake + +#============================================================================= +# Copyright 2011 Kitware, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= + +# Use the Cython executable that lives next to the Python executable +# if it is a local installation. + +if(PACKAGE_FIND_VERSION_MAJOR EQUAL 3) + set(CYTHON_EXE_NAMES cython3 cython.bat cython) + message(STATUS " Looking for Cython version 3") +else() + set(CYTHON_EXE_NAMES cython.bat cython cython3) +endif() + +if(PYTHONINTERP_FOUND) + get_filename_component( _python_path ${PYTHON_EXECUTABLE} PATH ) + find_program(CYTHON_EXECUTABLE + NAMES ${CYTHON_EXE_NAMES} + HINTS ${_python_path} + ) +else() + find_program(CYTHON_EXECUTABLE NAMES ${CYTHON_EXE_NAMES}) +endif() + +include( FindPackageHandleStandardArgs ) +find_package_handle_standard_args(Cython DEFAULT_MSG CYTHON_EXECUTABLE) + +if(CYTHON_FOUND) + message(STATUS "Found Cython (executable: ${CYTHON_EXECUTABLE})") + mark_as_advanced( CYTHON_EXECUTABLE ) +endif() + diff --git a/cmake/Modules/FindOpenBLAS.cmake b/cmake/Modules/FindOpenBLAS.cmake index a3a79ca..e325b9e 100644 --- a/cmake/Modules/FindOpenBLAS.cmake +++ b/cmake/Modules/FindOpenBLAS.cmake @@ -15,7 +15,6 @@ # specific language governing permissions and limitations # under the License. -file(TO_CMAKE_PATH "$ENV{OpenBLAS_HOME}" OpenBLAS_HOME) file(TO_CMAKE_PATH "$ENV{OpenBLAS}" OpenBLAS_DIR) SET(Open_BLAS_INCLUDE_SEARCH_PATHS @@ -31,6 +30,8 @@ SET(Open_BLAS_INCLUDE_SEARCH_PATHS ${PROJECT_SOURCE_DIR}/thirdparty/OpenBLAS/include ${OpenBLAS_HOME} ${OpenBLAS_HOME}/include + $ENV{OpenBLAS_HOME} + $ENV{OpenBLAS_HOME}/include ) SET(Open_BLAS_LIB_SEARCH_PATHS @@ -46,10 +47,12 @@ SET(Open_BLAS_LIB_SEARCH_PATHS /usr/local/opt/openblas/lib ${PROJECT_SOURCE_DIR}/3rdparty/OpenBLAS/lib ${PROJECT_SOURCE_DIR}/thirdparty/OpenBLAS/lib - ${OpenBLAS_DIR} - ${OpenBLAS_DIR}/lib + ${OpenBLAS_DIR} + ${OpenBLAS_DIR}/lib ${OpenBLAS_HOME} ${OpenBLAS_HOME}/lib + $ENV{OpenBLAS_HOME} + $ENV{OpenBLAS_HOME}/lib ) FIND_PATH(OpenBLAS_INCLUDE_DIR NAMES cblas.h PATHS ${Open_BLAS_INCLUDE_SEARCH_PATHS}) diff --git a/cmake/UseCython.cmake b/cmake/UseCython.cmake new file mode 100644 index 0000000..26969dd --- /dev/null +++ b/cmake/UseCython.cmake @@ -0,0 +1,390 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Define a function to create Cython modules. +# +# For more information on the Cython project, see http://cython.org/. +# "Cython is a language that makes writing C extensions for the Python language +# as easy as Python itself." +# +# This file defines a CMake function to build a Cython Python module. +# To use it, first include this file. +# +# include( UseCython ) +# +# Then call cython_add_module to create a module. +# +# cython_add_module( <module_name> <src1> <src2> ... <srcN> ) +# +# To create a standalone executable, the function +# +# cython_add_standalone_executable( <executable_name> [MAIN_MODULE src1] <src1> <src2> ... <srcN> ) +# +# To avoid dependence on Python, set the PYTHON_LIBRARY cache variable to point +# to a static library. If a MAIN_MODULE source is specified, +# the "if __name__ == '__main__':" from that module is used as the C main() method +# for the executable. If MAIN_MODULE, the source with the same basename as +# <executable_name> is assumed to be the MAIN_MODULE. +# +# Where <module_name> is the name of the resulting Python module and +# <src1> <src2> ... are source files to be compiled into the module, e.g. *.pyx, +# *.py, *.c, *.cxx, etc. A CMake target is created with name <module_name>. This can +# be used for target_link_libraries(), etc. +# +# The sample paths set with the CMake include_directories() command will be used +# for include directories to search for *.pxd when running the Cython complire. +# +# Cache variables that effect the behavior include: +# +# CYTHON_ANNOTATE +# CYTHON_NO_DOCSTRINGS +# CYTHON_FLAGS +# +# Source file properties that effect the build process are +# +# CYTHON_IS_CXX +# +# If this is set of a *.pyx file with CMake set_source_files_properties() +# command, the file will be compiled as a C++ file. +# +# See also FindCython.cmake + +#============================================================================= +# Copyright 2011 Kitware, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#============================================================================= + +# Configuration options. +set( CYTHON_ANNOTATE OFF + CACHE BOOL "Create an annotated .html file when compiling *.pyx." ) +set( CYTHON_NO_DOCSTRINGS OFF + CACHE BOOL "Strip docstrings from the compiled module." ) +set( CYTHON_FLAGS "" CACHE STRING + "Extra flags to the cython compiler." ) +mark_as_advanced( CYTHON_ANNOTATE CYTHON_NO_DOCSTRINGS CYTHON_FLAGS ) + +unset(PYTHONLIBS_FOUND) +unset(PYTHONINTERP_FOUND) +unset(CYTHON_FOUND) + +if(NOT python_libs_version) + message(STATUS "Looking for python dependencies, version: ${python_libs_version}") +endif() + +find_package(PythonInterp ${python_libs_version} REQUIRED) +if(PYTHONINTERP_FOUND) + message(STATUS "Python ${python_libs_version} executable: ${PYTHON_EXECUTABLE}") + find_package(PythonLibs ${python_libs_version} REQUIRED) + if(PYTHONLIBS_FOUND) + set(PYTHON_DEBUG_LIBRARY ${PYTHON_LIBRARY}) + set(PYTHON_DEBUG_LIBRARIES ${PYTHON_DEBUG_LIBRARIES}) + find_package(Cython ${python_libs_version} REQUIRED) + if(CYTHON_FOUND) + set(CYTHON${python_libs_version}_FOUND ${python_libs_version}) + message(STATUS " CYTHON${python_libs_version}_FOUND: ${CYTHON${python_libs_version}_FOUND}") + else() + message(WARNING " Could not find package: Cython") + endif() + else() + message(WARNING " Could not find package: PythonLibs") + endif() +else() + message(WARNING " Could not find package: PythonInterp") +endif() + +if(NOT CYTHON${python_libs_version}_FOUND) + message(WARNING " Could not find cython and/or dependencies for major version ${python_libs_version}") + return() +endif() + +message(STATUS "PYTHONLIBS_VERSION_STRING: ${PYTHONLIBS_VERSION_STRING}") +string(REPLACE "." ";" PYTHON_VERSION_LIST ${PYTHONLIBS_VERSION_STRING}) +list(GET PYTHON_VERSION_LIST 0 PYTHON_VERSION_MAJOR) +list(GET PYTHON_VERSION_LIST 1 PYTHON_VERSION_MINOR) +list(GET PYTHON_VERSION_LIST 1 PYTHON_VERSION_PATCH) + +if(NOT PYTHON_VERSION_MAJOR EQUAL ${python_libs_version}) + message(WARNING "Scripts found wrong python major version: ${PYTHON_VERSION_MAJOR} instead of ${python_libs_version}. This is most likely due to version ${python_libs_version} not being installed or not found") + unset(PYTHONLIBS_FOUND) + unset(PYTHONINTERP_FOUND) + unset(CYTHON_FOUND) + return() +endif() + +set( CYTHON_CXX_EXTENSION "cxx" ) +set( CYTHON_C_EXTENSION "c" ) + +# Create a *.c or *.cxx file from a *.pyx file. +# Input the generated file basename. The generate file will put into the variable +# placed in the "generated_file" argument. Finally all the *.py and *.pyx files. +function( compile_pyx _name c_cxx_output_subdir debug_output_dir generated_file ) + # Default to assuming all files are C. + set( cxx_arg "" ) + set( extension ${CYTHON_C_EXTENSION} ) + set( pyx_lang "C" ) + set( comment "Compiling Cython C source for ${_name}..." ) + + set( cython_include_directories "" ) + set( pxd_dependencies "" ) + set( pxi_dependencies "" ) + set( c_header_dependencies "" ) + set( pyx_locations "" ) + + #message(STATUS " compile_pyx( ${_name} ${generated_file} ${ARGN} )") + + foreach( pyx_file ${ARGN} ) + get_filename_component( pyx_file_basename "${pyx_file}" NAME_WE ) + + # Determine if it is a C or C++ file. + get_source_file_property( property_is_cxx ${pyx_file} CYTHON_IS_CXX ) + if( ${property_is_cxx} ) + set( cxx_arg "--cplus" ) + set( extension ${CYTHON_CXX_EXTENSION} ) + set( pyx_lang "CXX" ) + set( comment "Compiling Cython CXX source for ${_name}..." ) + endif() + + # Get the include directories. + get_source_file_property( pyx_location ${pyx_file} LOCATION ) + + get_filename_component( pyx_path ${pyx_location} PATH ) + + #get_directory_property( cmake_include_directories DIRECTORY ${pyx_path} INCLUDE_DIRECTORIES ) + + list( APPEND cython_include_directories ${cmake_include_directories} ) + list( APPEND pyx_locations "${pyx_location}" ) + + # Determine dependencies. + # Add the pxd file will the same name as the given pyx file. + unset( corresponding_pxd_file CACHE ) + find_file( corresponding_pxd_file ${pyx_file_basename}.pxd + PATHS "${pyx_path}" ${cmake_include_directories} + NO_DEFAULT_PATH ) + if( corresponding_pxd_file ) + list( APPEND pxd_dependencies "${corresponding_pxd_file}" ) + endif() + + # Look for included pxi files + file(STRINGS "${pyx_file}" include_statements REGEX "include +['\"]([^'\"]+).*") + foreach(statement ${include_statements}) + string(REGEX REPLACE "include +['\"]([^'\"]+).*" "\\1" pxi_file "${statement}") + unset(pxi_location CACHE) + find_file(pxi_location ${pxi_file} + PATHS "${pyx_path}" ${cmake_include_directories} NO_DEFAULT_PATH) + if (pxi_location) + list(APPEND pxi_dependencies ${pxi_location}) + get_filename_component( found_pyi_file_basename "${pxi_file}" NAME_WE ) + get_filename_component( found_pyi_path ${pxi_location} PATH ) + unset( found_pyi_pxd_file CACHE ) + find_file( found_pyi_pxd_file ${found_pyi_file_basename}.pxd + PATHS "${found_pyi_path}" ${cmake_include_directories} NO_DEFAULT_PATH ) + if (found_pyi_pxd_file) + list( APPEND pxd_dependencies "${found_pyi_pxd_file}" ) + endif() + endif() + endforeach() # for each include statement found + + # pxd files to check for additional dependencies. + set( pxds_to_check "${pyx_file}" "${pxd_dependencies}" ) + set( pxds_checked "" ) + set( number_pxds_to_check 1 ) + while( ${number_pxds_to_check} GREATER 0 ) + foreach( pxd ${pxds_to_check} ) + list( APPEND pxds_checked "${pxd}" ) + list( REMOVE_ITEM pxds_to_check "${pxd}" ) + + # check for C header dependencies + file( STRINGS "${pxd}" extern_from_statements + REGEX "cdef[ ]+extern[ ]+from.*$" ) + foreach( statement ${extern_from_statements} ) + # Had trouble getting the quote in the regex + string( REGEX REPLACE "cdef[ ]+extern[ ]+from[ ]+[\"]([^\"]+)[\"].*" "\\1" header "${statement}" ) + unset( header_location CACHE ) + find_file( header_location ${header} PATHS ${cmake_include_directories} ) + if( header_location ) + list( FIND c_header_dependencies "${header_location}" header_idx ) + if( ${header_idx} LESS 0 ) + list( APPEND c_header_dependencies "${header_location}" ) + endif() + endif() + endforeach() + + # check for pxd dependencies + + # Look for cimport statements. + set( module_dependencies "" ) + file( STRINGS "${pxd}" cimport_statements REGEX cimport ) + foreach( statement ${cimport_statements} ) + if( ${statement} MATCHES from ) + string( REGEX REPLACE "from[ ]+([^ ]+).*" "\\1" module "${statement}" ) + else() + string( REGEX REPLACE "cimport[ ]+([^ ]+).*" "\\1" module "${statement}" ) + endif() + list( APPEND module_dependencies ${module} ) + endforeach() + list( REMOVE_DUPLICATES module_dependencies ) + # Add the module to the files to check, if appropriate. + foreach( module ${module_dependencies} ) + unset( pxd_location CACHE ) + find_file( pxd_location ${module}.pxd + PATHS "${pyx_path}" ${cmake_include_directories} NO_DEFAULT_PATH ) + if( pxd_location ) + list( FIND pxds_checked ${pxd_location} pxd_idx ) + if( ${pxd_idx} LESS 0 ) + list( FIND pxds_to_check ${pxd_location} pxd_idx ) + if( ${pxd_idx} LESS 0 ) + list( APPEND pxds_to_check ${pxd_location} ) + list( APPEND pxd_dependencies ${pxd_location} ) + endif() # if it is not already going to be checked + endif() # if it has not already been checked + endif() # if pxd file can be found + endforeach() # for each module dependency discovered + endforeach() # for each pxd file to check + list( LENGTH pxds_to_check number_pxds_to_check ) + endwhile() + + + + endforeach() # pyx_file + + # Set additional flags. + if( CYTHON_ANNOTATE ) + set( annotate_arg "--annotate" ) + endif() + + if( CYTHON_NO_DOCSTRINGS ) + set( no_docstrings_arg "--no-docstrings" ) + endif() + + if( "${CMAKE_BUILD_TYPE}" STREQUAL "Debug" OR + "${CMAKE_BUILD_TYPE}" STREQUAL "RelWithDebInfo" ) + set( cython_debug_arg + #"--gdb" + --gdb-outdir ${debug_output_dir} + ) + endif() + + if( "${PYTHONLIBS_VERSION_STRING}" MATCHES "^2." ) + set( version_arg "-2" ) + elseif( "${PYTHONLIBS_VERSION_STRING}" MATCHES "^3." ) + set( version_arg "-3" ) + else() + set( version_arg ) + endif() + + # Include directory arguments. + list( REMOVE_DUPLICATES cython_include_directories ) + set( include_directory_arg "" ) + foreach( _include_dir ${cython_include_directories} ) + set( include_directory_arg ${include_directory_arg} "-I" "${_include_dir}" ) + endforeach() + + # Determining generated file name. + set( _generated_file "${CMAKE_CURRENT_BINARY_DIR}/${c_cxx_output_subdir}/${_name}.${extension}" ) + set_source_files_properties( ${_generated_file} PROPERTIES GENERATED TRUE ) + set( ${generated_file} ${_generated_file} PARENT_SCOPE ) + + list( REMOVE_DUPLICATES pxd_dependencies ) + list( REMOVE_DUPLICATES c_header_dependencies ) + + # Add the command to run the compiler. + add_custom_command( OUTPUT ${_generated_file} + COMMAND ${CYTHON_EXECUTABLE} + ARGS ${cxx_arg} ${include_directory_arg} ${version_arg} + ${annotate_arg} ${no_docstrings_arg} ${cython_debug_arg} ${CYTHON_FLAGS} + --output-file ${_generated_file} ${pyx_locations} + DEPENDS ${pyx_locations} ${pxd_dependencies} ${pxi_dependencies} + IMPLICIT_DEPENDS ${pyx_lang} ${c_header_dependencies} + WORKING_DIRECTORY ${c_cxx_output_subdir} + COMMENT ${comment} + ) + + # Remove their visibility to the user. + set( corresponding_pxd_file "" CACHE INTERNAL "" ) + set( header_location "" CACHE INTERNAL "" ) + set( pxd_location "" CACHE INTERNAL "" ) +endfunction() + +# cython_add_module( <name> src1 src2 ... srcN ) +# Build the Cython Python module. +function( cython_add_module _name c_cxx_output_subdir debug_output_dir) + set( pyx_module_sources "" ) + set( other_module_sources "" ) + foreach( _file ${ARGN} ) + if( ${_file} MATCHES ".*\\.py[x]?$" ) + list( APPEND pyx_module_sources ${_file} ) + else() + list( APPEND other_module_sources ${_file} ) + endif() + endforeach() + compile_pyx( ${_name} ${c_cxx_output_subdir} ${debug_output_dir} generated_file ${pyx_module_sources}) + include_directories( ${PYTHON_INCLUDE_DIRS} ) + python_add_module( ${_name} ${generated_file} ${other_module_sources} ) + if( APPLE ) + set_target_properties( ${_name} PROPERTIES LINK_FLAGS "-undefined dynamic_lookup" ) + else() + target_link_libraries( ${_name} ${PYTHON_LIBRARIES} ) + endif() +endfunction() + +include( CMakeParseArguments ) +# cython_add_standalone_executable( _name [MAIN_MODULE src3.py] src1 src2 ... srcN ) +# Creates a standalone executable the given sources. +function( cython_add_standalone_executable _name debug_output_dir) + set( pyx_module_sources "" ) + set( other_module_sources "" ) + set( main_module "" ) + cmake_parse_arguments( cython_arguments "" "MAIN_MODULE" "" ${ARGN} ) + include_directories( ${PYTHON_INCLUDE_DIRS} ) + foreach( _file ${cython_arguments_UNPARSED_ARGUMENTS} ) + if( ${_file} MATCHES ".*\\.py[x]?$" ) + get_filename_component( _file_we ${_file} NAME_WE ) + if( "${_file_we}" STREQUAL "${_name}" ) + set( main_module "${_file}" ) + elseif( NOT "${_file}" STREQUAL "${cython_arguments_MAIN_MODULE}" ) + set( PYTHON_MODULE_${_file_we}_static_BUILD_SHARED OFF ) + compile_pyx( "${_file_we}_static" generated_file ${debug_output_dir} "${_file}" ) + list( APPEND pyx_module_sources "${generated_file}" ) + endif() + else() + list( APPEND other_module_sources ${_file} ) + endif() + endforeach() + + if( cython_arguments_MAIN_MODULE ) + set( main_module ${cython_arguments_MAIN_MODULE} ) + endif() + if( NOT main_module ) + message( FATAL_ERROR "main module not found." ) + endif() + get_filename_component( main_module_we "${main_module}" NAME_WE ) + set( CYTHON_FLAGS ${CYTHON_FLAGS} --embed ) + compile_pyx( "${main_module_we}_static" generated_file ${main_module} ) + add_executable( ${_name} ${generated_file} ${pyx_module_sources} ${other_module_sources} ) + target_link_libraries( ${_name} ${PYTHON_LIBRARIES} ${pyx_module_libs} ) +endfunction() diff --git a/python/mxnet/_cy3/__init__.py b/python/mxnet/cython/__init__.py similarity index 93% copy from python/mxnet/_cy3/__init__.py copy to python/mxnet/cython/__init__.py index 44dcca5..13a8339 100644 --- a/python/mxnet/_cy3/__init__.py +++ b/python/mxnet/cython/__init__.py @@ -14,5 +14,3 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - -"""Namespace for cython generated modules for python3""" diff --git a/python/mxnet/cython/base.pyi b/python/mxnet/cython/base.pyi index d73e1a7..3963b37 100644 --- a/python/mxnet/cython/base.pyi +++ b/python/mxnet/cython/base.pyi @@ -1,4 +1,4 @@ -from ..base import MXNetError +from ...base import MXNetError from libcpp.vector cimport vector from libcpp.string cimport string diff --git a/python/mxnet/_cy3/README b/python/mxnet/cython/cy2/README similarity index 100% rename from python/mxnet/_cy3/README rename to python/mxnet/cython/cy2/README diff --git a/python/mxnet/_cy2/__init__.py b/python/mxnet/cython/cy2/__init__.py similarity index 100% copy from python/mxnet/_cy2/__init__.py copy to python/mxnet/cython/cy2/__init__.py diff --git a/python/mxnet/_cy2/README b/python/mxnet/cython/cy3/README similarity index 100% rename from python/mxnet/_cy2/README rename to python/mxnet/cython/cy3/README diff --git a/python/mxnet/_cy3/__init__.py b/python/mxnet/cython/cy3/__init__.py similarity index 100% copy from python/mxnet/_cy3/__init__.py copy to python/mxnet/cython/cy3/__init__.py diff --git a/python/mxnet/cython/mxcython.pyx b/python/mxnet/cython/mxcython.pyx new file mode 100644 index 0000000..96a0034 --- /dev/null +++ b/python/mxnet/cython/mxcython.pyx @@ -0,0 +1,86 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import absolute_import as _abs + +import sys as _sys +import ctypes as _ctypes +import numpy as np +import time +from ...ndarray_doc import _build_doc +from libc.stdint cimport uint32_t, int64_t + +include "./base.pyi" + +# C API functions +cdef extern from "../../../src/cython/cython_util.h": + int CythonPrintFromCPP(const char *foo); + int Printf(const char *fmt, ...); + int TrivialCPPCall(int var); + unsigned long long TimeInMilliseconds(); + +# C++ Rectangle class +cdef extern from "../../../src/cython/cython_util.h" namespace "shapes": + cdef cppclass Rectangle: + Rectangle() except + + Rectangle(int, int, int, int) except + + int x0, y0, x1, y1 + int getArea() + void getSize(int* width, int* height) + void move(int, int) + + +# Cython class: CythonTestClass +cdef class CythonTestClass: + """Symbol is symbolic graph.""" + # handle for symbolic operator. + cdef int cwritable + + def __init__(self): + self.cwritable = 99 + + def print_something(self, str the_string): + print('BEFORE CythonPrintFromCPP') + CALL(CythonPrintFromCPP("This is from C++")) + print('AFTER CythonPrintFromCPP') + print('CythonTestClass::print_something( {} )'.format(the_string)) + +def test_cpp_class(): + cdef int recArea + rec_ptr = new Rectangle(1, 2, 3, 4) + try: + recArea = rec_ptr.getArea() + Printf("Printf() call: Area: %d\n", recArea) + finally: + del rec_ptr # delete heap allocated object + +def test_perf(int count, int make_c_call): + cdef unsigned long long start = TimeInMilliseconds() + cdef int foo = 0 + cdef int i = 0 + while i < count: + foo += i + if foo > count: + foo = 0 + if make_c_call != 0: + TrivialCPPCall(0) + i += 1 + cdef unsigned long long stop = TimeInMilliseconds() + Printf("CYTHON: %d items took %f seconds\n", count, float(stop - start)/1000) + +def print_pi(terms): + print(float(0.0)) diff --git a/python/mxnet/_cy2/__init__.py b/python/mxnet/ndarray/cy2/__init__.py similarity index 100% copy from python/mxnet/_cy2/__init__.py copy to python/mxnet/ndarray/cy2/__init__.py diff --git a/python/mxnet/_cy3/__init__.py b/python/mxnet/ndarray/cy3/__init__.py similarity index 100% copy from python/mxnet/_cy3/__init__.py copy to python/mxnet/ndarray/cy3/__init__.py diff --git a/python/mxnet/cython/ndarray.pyx b/python/mxnet/ndarray/ndarray.pyx similarity index 98% rename from python/mxnet/cython/ndarray.pyx rename to python/mxnet/ndarray/ndarray.pyx index 319dc49..57e207d 100644 --- a/python/mxnet/cython/ndarray.pyx +++ b/python/mxnet/ndarray/ndarray.pyx @@ -20,10 +20,10 @@ from __future__ import absolute_import as _abs import sys as _sys import ctypes as _ctypes import numpy as np -from ..ndarray_doc import _build_doc +from ...ndarray_doc import _build_doc from libc.stdint cimport uint32_t, int64_t -include "./base.pyi" +include "../cython/base.pyi" cdef class NDArrayBase: """Symbol is symbolic graph.""" diff --git a/python/mxnet/_cy2/__init__.py b/python/mxnet/symbol/cy2/__init__.py similarity index 100% rename from python/mxnet/_cy2/__init__.py rename to python/mxnet/symbol/cy2/__init__.py diff --git a/python/mxnet/_cy3/__init__.py b/python/mxnet/symbol/cy3/__init__.py similarity index 100% copy from python/mxnet/_cy3/__init__.py copy to python/mxnet/symbol/cy3/__init__.py diff --git a/python/mxnet/cython/symbol.pyx b/python/mxnet/symbol/symbol.pyx similarity index 96% rename from python/mxnet/cython/symbol.pyx rename to python/mxnet/symbol/symbol.pyx index 1bdea6c..c3062d4 100644 --- a/python/mxnet/cython/symbol.pyx +++ b/python/mxnet/symbol/symbol.pyx @@ -22,11 +22,11 @@ import ctypes as _ctypes import numpy as _numpy from numbers import Number as _Number -from ..name import NameManager -from ..attribute import AttrScope -from ..symbol_doc import _build_doc +from ...name import NameManager +from ...attribute import AttrScope +from ...symbol_doc import _build_doc -include "./base.pyi" +include "../cython/base.pyi" cdef class SymbolBase: """Symbol is symbolic graph.""" diff --git a/python/setup.py b/python/setup.py index cf94adf..320001f 100644 --- a/python/setup.py +++ b/python/setup.py @@ -31,10 +31,13 @@ else: kwargs = {'install_requires': ['numpy<=1.13.3,>=1.8.2', 'requests==2.18.4', 'graphviz==0.8.1'], 'zip_safe': False} from setuptools import find_packages -with_cython = False +with_cython = True if '--with-cython' in sys.argv: with_cython = True sys.argv.remove('--with-cython') +if '--without-cython' in sys.argv: + with_cython = False + sys.argv.remove('--without-cython') # We can not import `mxnet.info.py` in setup.py directly since mxnet/__init__.py # Will be invoked which introduces dependences diff --git a/python/mxnet/_cy3/__init__.py b/src/cython/cy2/CMakeLists.txt similarity index 61% copy from python/mxnet/_cy3/__init__.py copy to src/cython/cy2/CMakeLists.txt index 44dcca5..da67ba6 100644 --- a/python/mxnet/_cy3/__init__.py +++ b/src/cython/cy2/CMakeLists.txt @@ -15,4 +15,19 @@ # specific language governing permissions and limitations # under the License. -"""Namespace for cython generated modules for python3""" +cmake_minimum_required(VERSION 3.0.2) + +project(cy2 C CXX) + +include(${CMAKE_CURRENT_SOURCE_DIR}/../../../cmake/CythonUtil.cmake) + +set(cython_modules "") + +mxnet_build_cython_module(2 cython_modules) +#message(STATUS "cython_modules: ${cython_modules}") +#if(INSTALL_CYTHON_INPLACE) +# message(STATUS "Installing cython in-place") +# cython_install_into_source_dir("${MXNET_BINARY_DIR}/python" "${MXNET_ROOT_DIR}/python") +#else() +# message(STATUS "Not installing cython in-place") +#endif() diff --git a/python/mxnet/_cy3/__init__.py b/src/cython/cy3/CMakeLists.txt similarity index 61% copy from python/mxnet/_cy3/__init__.py copy to src/cython/cy3/CMakeLists.txt index 44dcca5..e5df735 100644 --- a/python/mxnet/_cy3/__init__.py +++ b/src/cython/cy3/CMakeLists.txt @@ -15,4 +15,19 @@ # specific language governing permissions and limitations # under the License. -"""Namespace for cython generated modules for python3""" +cmake_minimum_required(VERSION 3.0.2) + +project(cy3 C CXX) + +include(${CMAKE_CURRENT_SOURCE_DIR}/../../../cmake/CythonUtil.cmake) + +set(cython_modules "") + +mxnet_build_cython_module(3 cython_modules) +#message(STATUS "cython_modules: ${cython_modules}") +#if(INSTALL_CYTHON_INPLACE) +# message(STATUS "Installing cython in-place") +# cython_install_into_source_dir("${MXNET_BINARY_DIR}/python" "${MXNET_ROOT_DIR}/python") +#else() +# message(STATUS "Not installing cython in-place") +#endif() diff --git a/src/cython/cython_util.cc b/src/cython/cython_util.cc new file mode 100644 index 0000000..7aeddbd --- /dev/null +++ b/src/cython/cython_util.cc @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include <iostream> +#include <cstdarg> +#include <sys/time.h> +#include <chrono> +#include "./cython_util.h" + +extern "C" int CythonPrintFromCPP(const char *foo) { + if(foo) { + std::cout << foo << std::endl << std::flush; + } + return 0; +} + +extern "C" int Printf(const char *fmt, ...) { + va_list args; + va_start(args, fmt); + const int res = vprintf(fmt, args); + va_end(args); + return res; +} + +extern "C" int TrivialCPPCall(int var) { + static int static_var = 0; + static_var = var; + return static_var; +} + +using Tick = std::chrono::high_resolution_clock::time_point; +static inline Tick Now() { return std::chrono::high_resolution_clock::now(); } + +static const Tick _app_start_time = Now(); + +static inline uint64_t GetDurationInNanoseconds(const Tick &t1, const Tick &t2) { + return static_cast<uint64_t>( + std::chrono::duration_cast<std::chrono::nanoseconds>(t2 - t1).count()); +} + +static inline uint64_t GetDurationInNanoseconds(const Tick &since) { + return GetDurationInNanoseconds(since, Now()); +} + +constexpr size_t SLEEP_DURATION = 500; +constexpr size_t TIMER_PERIOD = 10; // Ideal is 50 periods occur +constexpr size_t MIN_COUNT_WHILE_SLEEPING = 10; +constexpr size_t MAX_COUNT_WHILE_SLEEPING = 150; + +static inline size_t GetDurationInMilliseconds(const Tick& start_time) { + return static_cast<size_t>(GetDurationInNanoseconds(start_time)/1000/1000); +} + + +extern "C" uint64_t TimeInMilliseconds() { + return GetDurationInMilliseconds(_app_start_time); +} + +namespace shapes { + +Rectangle::Rectangle() { } + +Rectangle::Rectangle(int X0, int Y0, int X1, int Y1) { + x0 = X0; + y0 = Y0; + x1 = X1; + y1 = Y1; +} + +Rectangle::~Rectangle() { } + +int Rectangle::getArea() { + return (x1 - x0) * (y1 - y0); +} + +void Rectangle::getSize(int *width, int *height) { + (*width) = x1 - x0; + (*height) = y1 - y0; +} + +void Rectangle::move(int dx, int dy) { + x0 += dx; + y0 += dy; + x1 += dx; + y1 += dy; +} + +} // namespace shapes diff --git a/src/cython/cython_util.h b/src/cython/cython_util.h new file mode 100644 index 0000000..8d519cf --- /dev/null +++ b/src/cython/cython_util.h @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#ifndef MXNET_CYTHON_CYTHON_UTIL_H_ +#define MXNET_CYTHON_CYTHON_UTIL_H_ + +/*! \brief Inhibit C++ name-mangling for MXNet functions. */ +#ifdef __cplusplus +extern "C" { +#endif // __cplusplus + +int CythonPrintFromCPP(const char *foo); +int Printf(const char *fmt, ...); +int TrivialCPPCall(int var); +uint64_t TimeInMilliseconds(); + +#ifdef __cplusplus +} +#endif // __cplusplus + +namespace shapes { + +class Rectangle { + public: + int x0, y0, x1, y1; + Rectangle(); + Rectangle(int x0, int y0, int x1, int y1); + ~Rectangle(); + int getArea(); + void getSize(int* width, int* height); + void move(int dx, int dy); +}; + +} // namespace shapes + +#endif // MXNET_CYTHON_CYTHON_UTIL_H_ diff --git a/tests/python/unittest/test_cython.py b/tests/python/unittest/test_cython.py new file mode 100644 index 0000000..d9ccd7a --- /dev/null +++ b/tests/python/unittest/test_cython.py @@ -0,0 +1,75 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Debugging help +# http://grapsus.net/blog/post/Low-level-Python-debugging-with-GDB + +from __future__ import print_function +import sys +import time +from mxnet.base import _LIB + +try: + if sys.version_info >= (3, 0): + import mxnet.cython.cy3.mxcython as mxc + import mxnet.ndarray.cy3.ndarray as ndcy + import mxnet.symbol.cy3.symbol as symcy + else: + import mxnet.cython.cy2.mxcython as mxc + import mxnet.ndarray.cy2.ndarray as ndcy + import mxnet.symbol.cy2.symbol as symcy +except: + # No cython found + print('Unable to load cython modules') + exit(1) + +def test_basic_cython(): + print('ENTER test_basic_cython') + myclass = mxc.CythonTestClass() + for terms in 5, 9, 23, 177, 1111, 33333, 555555: + sys.stdout.write('{0:10} terms: '.format(terms)) + mxc.print_pi(terms) + myclass.print_something('Something') + print('EXIT test_basic_cython') + + # Test using a C++ class' + mxc.test_cpp_class() + mxc.test_perf(10, 1) + + +def test_perf(count, make_c_call): + start = _LIB.TimeInMilliseconds() + foo = 0 + i = 0 + while i < count: + foo += i + if foo > count: + foo = 0 + if make_c_call != 0: + _LIB.TrivialCPPCall(0) + i += 1 + stop = _LIB.TimeInMilliseconds() + print("PYTHON: {} items took {} seconds".format(count, float(stop - start)/1000)) + +if __name__ == '__main__': + # import nose + # nose.runmodule() + # test_perf(100000000, 0) + # mxc.test_perf(100000000, 0) + # test_perf(100000000, 1) + # mxc.test_perf(100000000, 1) + test_basic_cython() diff --git a/python/mxnet/_cy3/__init__.py b/tools/cython/clean_cython.sh old mode 100644 new mode 100755 similarity index 79% copy from python/mxnet/_cy3/__init__.py copy to tools/cython/clean_cython.sh index 44dcca5..7f2d5f8 --- a/python/mxnet/_cy3/__init__.py +++ b/tools/cython/clean_cython.sh @@ -1,3 +1,4 @@ +#!/bin/sh # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -14,5 +15,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - -"""Namespace for cython generated modules for python3""" +ROOTDIR=$(dirname $0)/.. +for i in $(find $ROOTDIR/python/mxnet -type f -name "*.so"); do rm -f $i; done +for i in $(find $ROOTDIR/python/mxnet -type d -name "cython_debug"); do rm -rf $i; done diff --git a/python/mxnet/_cy3/__init__.py b/tools/cython/cydb2 old mode 100644 new mode 100755 similarity index 68% copy from python/mxnet/_cy3/__init__.py copy to tools/cython/cydb2 index 44dcca5..5157a26 --- a/python/mxnet/_cy3/__init__.py +++ b/tools/cython/cydb2 @@ -1,3 +1,4 @@ +#!/bin/bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,4 +16,12 @@ # specific language governing permissions and limitations # under the License. -"""Namespace for cython generated modules for python3""" +ROOTDIR=$(realpath $(dirname $0)) +cd $ROOTDIR/../.. +if [ -d ../../cmake-build-debug/mxnet/cython/cy2 ]; then + cygdb ../../cmake-build-debug/mxnet/cython/cy2 -- $@ +elif [ -d ../cmake-build-debug/mxnet/cython/cy2 ]; then + cygdb ../cmake-build-debug/mxnet/cython/cy2 -- $@ +else + echo "$0: Don't know where to find cythoin debug info" +fi diff --git a/python/mxnet/_cy3/__init__.py b/tools/cython/cydb3 old mode 100644 new mode 100755 similarity index 68% rename from python/mxnet/_cy3/__init__.py rename to tools/cython/cydb3 index 44dcca5..62a346a --- a/python/mxnet/_cy3/__init__.py +++ b/tools/cython/cydb3 @@ -1,3 +1,4 @@ +#!/bin/bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -15,4 +16,12 @@ # specific language governing permissions and limitations # under the License. -"""Namespace for cython generated modules for python3""" +ROOTDIR=$(realpath $(dirname $0)) +cd $ROOTDIR/../.. +if [ -d ../../cmake-build-debug/mxnet/cython/cy3 ]; then + cygdb ../../cmake-build-debug/mxnet/cython/cy3 -- $@ +elif [ -d ../cmake-build-debug/mxnet/cython/cy3 ]; then + cygdb ../cmake-build-debug/mxnet/cython/cy3 -- $@ +else + echo "$0: Don't know where to find cythoin debug info" +fi -- To stop receiving notification emails like this one, please contact cjolivie...@apache.org.