Merge pull request #1092 from gangliao/python_default

Check Python if system already equipped one

.travis.yml

@@ -21,13 +21,14 @@ addons:
     packages:
       - gcc-4.8
       - g++-4.8
-      - wget
       - git
       - build-essential
       - libatlas-base-dev
       - python
       - python-pip
       - python2.7-dev
+      - python-numpy
+      - python-wheel
       - curl
       - swig
       - graphviz
@@ -48,8 +49,7 @@ before_install:
     fi
   - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then paddle/scripts/travis/before_install.osx.sh; fi
   - if [[ "$JOB" == "PRE_COMMIT" ]]; then sudo ln -s /usr/bin/clang-format-3.8 /usr/bin/clang-format; fi
-  - pip install --upgrade pip
-  - pip install wheel protobuf sphinx recommonmark sphinx_rtd_theme virtualenv pre-commit requests==2.9.2 LinkChecker
+  - pip install numpy wheel protobuf sphinx recommonmark sphinx_rtd_theme virtualenv pre-commit requests==2.9.2 LinkChecker
 script:
   - paddle/scripts/travis/main.sh
 notifications:

CMakeLists.txt

@@ -28,21 +28,21 @@ include(system)
 include(simd)
 
 ###################### Configurations ############################
-option(WITH_DSO "Compile PaddlePaddle with dynamic linked libraries" ON)
-option(WITH_GPU "Compile PaddlePaddle with gpu" ${CUDA_FOUND})
-option(WITH_DOUBLE "Compile PaddlePaddle with double precision, otherwise use single precision" OFF)
-option(WITH_AVX "Compile PaddlePaddle with avx intrinsics" ${AVX_FOUND})
+option(WITH_GPU         "Compile PaddlePaddle with NVIDIA GPU"          ${CUDA_FOUND})
+option(WITH_AVX         "Compile PaddlePaddle with AVX intrinsics"      ${AVX_FOUND})
+option(WITH_DSO         "Compile PaddlePaddle with dynamic linked CUDA" ON)
+option(WITH_TESTING     "Compile PaddlePaddle with unit testing"        ON)
+option(WITH_SWIG_PY     "Compile PaddlePaddle with inference api"       ON)
+option(WITH_STYLE_CHECK "Compile PaddlePaddle with style check"         ON)
 option(WITH_PYTHON      "Compile PaddlePaddle with python interpreter"  ON)
-option(WITH_STYLE_CHECK "Style Check for PaddlePaddle" ON)
-option(WITH_RDMA "Compile PaddlePaddle with rdma support" OFF)
-option(WITH_TIMER "Compile PaddlePaddle use timer" OFF)
-option(WITH_PROFILER "Compile PaddlePaddle use gpu profiler" OFF)
-option(WITH_TESTING "Compile and run unittest for PaddlePaddle" ON)
+option(WITH_DOUBLE      "Compile PaddlePaddle with double precision"    OFF)
+option(WITH_RDMA        "Compile PaddlePaddle with RDMA support"        OFF)
+option(WITH_TIMER       "Compile PaddlePaddle with stats timer"         OFF)
+option(WITH_PROFILER    "Compile PaddlePaddle with GPU profiler"        OFF)
 option(WITH_DOC         "Compile PaddlePaddle with documentation"       OFF)
-option(WITH_SWIG_PY "Compile PaddlePaddle with py PaddlePaddle prediction api" ON)
-option(ON_TRAVIS "Running test on travis-ci or not." OFF)
-option(ON_COVERALLS "Generating code coverage data on coveralls or not." OFF)
-option(COVERALLS_UPLOAD "Uploading the generated coveralls json." ON)
+option(ON_COVERALLS     "Compile PaddlePaddle with code coverage"       OFF)
+option(COVERALLS_UPLOAD "Package code coverage data to coveralls"       OFF)
+option(ON_TRAVIS        "Exclude special unit test on Travis CI"        OFF)
 
 include(external/zlib)      # download, build, install zlib
 include(external/gflags)    # download, build, install gflags
@@ -63,7 +63,6 @@ include(flags)              # set paddle compile flags
 include(cudnn)              # set cudnn libraries
 include(version)            # set PADDLE_VERSION
 include(coveralls)          # set code coverage
-include(python_module)      # set python module
 
 include(configure)          # add paddle env configuration
 

cmake/configure.cmake

@@ -12,6 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+if(NOT WITH_PYTHON)
+    add_definitions(-DPADDLE_NO_PYTHON)
+endif(NOT WITH_PYTHON)
+
 if(WITH_DSO)
     add_definitions(-DPADDLE_USE_DSO)
 endif(WITH_DSO)

cmake/external/python.cmake

@@ -13,8 +13,20 @@
 # limitations under the License.
 
 INCLUDE(ExternalProject)
+INCLUDE(python_module)
 
+FIND_PACKAGE(PythonInterp 2.7)
+FIND_PACKAGE(PythonLibs 2.7)
 
+SET(py_env "")
+
+IF(PYTHONLIBS_FOUND AND PYTHONINTERP_FOUND)
+    find_python_module(pip REQUIRED)
+    find_python_module(numpy REQUIRED)
+    find_python_module(wheel REQUIRED)
+    find_python_module(google.protobuf REQUIRED)
+    FIND_PACKAGE(NumPy REQUIRED)
+ELSE(PYTHONLIBS_FOUND AND PYTHONINTERP_FOUND)
     ##################################### PYTHON ########################################
     SET(PYTHON_SOURCES_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/python)
     SET(PYTHON_INSTALL_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/install/python)
@@ -36,13 +48,6 @@ ELSE()
         MESSAGE(FATAL_ERROR "Unknown system !")
     ENDIF()
 
-SET(py_env
-    PATH=${PYTHON_INSTALL_DIR}/bin/:$ENV{PATH}
-    PYTHONHOME=${PYTHON_INSTALL_DIR}
-    PYTHONPATH=${PYTHON_INSTALL_DIR}/lib:${PYTHON_INSTALL_DIR}/lib/python2.7:${PY_SITE_PACKAGES_PATH})
-
-INCLUDE_DIRECTORIES(${PYTHON_INCLUDE_DIR})
-
     IF(APPLE)
         LIST(APPEND EXTERNAL_PROJECT_OPTIONAL_CMAKE_ARGS
             -DCMAKE_BUILD_WITH_INSTALL_RPATH:BOOL=ON
@@ -82,6 +87,11 @@ ExternalProject_Add(python
             ${EXTERNAL_PROJECT_OPTIONAL_CMAKE_ARGS}
         DEPENDS zlib
     )
+
+    SET(py_env
+        PATH=${PYTHON_INSTALL_DIR}/bin
+        PYTHONHOME=${PYTHON_INSTALL_DIR}
+        PYTHONPATH=${PYTHON_INSTALL_DIR}/lib:${PYTHON_INSTALL_DIR}/lib/python2.7:${PY_SITE_PACKAGES_PATH})
     ####################################################################################
 
     ##################################### SETUPTOOLS ###################################
@@ -155,8 +165,6 @@ ELSE(WIN32)
         ENDFOREACH()
     ENDIF(WIN32)
 
-INCLUDE_DIRECTORIES(${PYTHON_NUMPY_INCLUDE_DIR})
-
     ExternalProject_Add(numpy
         ${EXTERNAL_PROJECT_LOG_ARGS}
         GIT_REPOSITORY      https://github.com/numpy/numpy.git
@@ -196,9 +204,19 @@ ExternalProject_Add(python-protobuf
         BUILD_IN_SOURCE       1
         PATCH_COMMAND         ""
         CONFIGURE_COMMAND     ""
-    BUILD_COMMAND         env PATH=${PROTOBUF_INSTALL_DIR}/bin:$ENV{PATH} ${py_env} ${PYTHON_EXECUTABLE} setup.py build
-    INSTALL_COMMAND       env PATH=${PROTOBUF_INSTALL_DIR}/bin:$ENV{PATH} ${py_env} ${PYTHON_EXECUTABLE} setup.py install
+        BUILD_COMMAND         env ${py_env} ${PYTHON_EXECUTABLE} setup.py build
+        INSTALL_COMMAND       env ${py_env} ${PYTHON_EXECUTABLE} setup.py install
         DEPENDS               python setuptools six
     )
+    ####################################################################################
+
+    LIST(APPEND external_project_dependencies python setuptools six cython wheel python-protobuf numpy)
+
+ENDIF(PYTHONLIBS_FOUND AND PYTHONINTERP_FOUND)
+
+INCLUDE_DIRECTORIES(${PYTHON_INCLUDE_DIR})
+INCLUDE_DIRECTORIES(${PYTHON_NUMPY_INCLUDE_DIR})
 
-LIST(APPEND external_project_dependencies python setuptools six cython numpy wheel python-protobuf)
+MESSAGE("[Paddle] Python Executable: ${PYTHON_EXECUTABLE}")
+MESSAGE("[Paddle] Python Include: ${PYTHON_INCLUDE_DIRS}")
+MESSAGE("[Paddle] Python Libraries: ${PYTHON_LIBRARIES}")

paddle/api/CMakeLists.txt

@@ -94,6 +94,7 @@ add_dependencies(python_api_wheel python_swig_sources
   paddle_cuda)
 
 if(WITH_TESTING)
+    IF(NOT PY_PIP_FOUND)
         SET(PIP_SOURCES_DIR ${PYTHON_SOURCES_DIR}/pip)
         ExternalProject_Add(pip
             ${EXTERNAL_PROJECT_LOG_ARGS}
@@ -106,5 +107,6 @@ if(WITH_TESTING)
             BUILD_IN_SOURCE     1
             DEPENDS python setuptools python_api_wheel
         )
+    ENDIF()
     add_subdirectory(test)
 endif()

paddle/scripts/travis/before_install.osx.sh

@@ -1,4 +1,6 @@
 #!/bin/bash
 brew update
 brew tap homebrew/science
-brew install openblas md5sha1sum 
+brew install python
+sudo pip install --upgrade protobuf
+brew install swig openblas md5sha1sum protobuf

paddle/scripts/travis/build_and_test.sh

@@ -1,26 +1,19 @@
 #!/bin/bash
 source ./common.sh
 
-python -c 'import pip; print(pip.pep425tags.get_supported())'
-
-if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
-  CMAKE_EXTRA="-DWITH_SWIG_PY=OFF"
-else
-  CMAKE_EXTRA="-DWITH_SWIG_PY=ON"
-fi
-
-cmake .. -DWITH_GPU=OFF -DWITH_DOC=OFF -DWITH_TESTING=ON -DON_TRAVIS=ON -DON_COVERALLS=ON ${CMAKE_EXTRA}
-
 NPROC=1
 if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
+  export PYTHONPATH=/opt/python/2.7.12/lib/python2.7/site-packages
+  export PYTHONHOME=/opt/python/2.7.12
+  export PATH=/opt/python/2.7.12/bin:${PATH}
+  cmake .. -DON_TRAVIS=ON -DON_COVERALLS=ON -DCOVERALLS_UPLOAD=ON
   NRPOC=`nproc`
   make -j $NPROC
   make coveralls
   sudo make install
 elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
+  export PYTHONPATH=/usr/local/lib/python2.7/site-packages
+  cmake .. -DON_TRAVIS=ON -DON_COVERALLS=ON -DCOVERALLS_UPLOAD=ON -DWITH_SWIG_PY=ON
   NPROC=`sysctl -n hw.ncpu`
   make -j $NPROC
-  env CTEST_OUTPUT_ON_FAILURE=1 make test ARGS="-j $NPROC"
-  sudo make install
-  sudo paddle version
 fi

paddle/utils/PythonUtil.cpp.in

@@ -195,8 +195,10 @@ extern const char enable_virtualenv_py[];
 }
 void initPython(int argc, char** argv) {
 #ifndef PADDLE_NO_PYTHON
-  char PythonHome[] = "@PYTHON_INSTALL_DIR@"; // NOLINT
-  Py_SetPythonHome(PythonHome);
+  char pyHome[] = "@PYTHON_INSTALL_DIR@"; // NOLINT
+  if (strlen(pyHome)) {
+    Py_SetPythonHome(pyHome);
+  }
   Py_SetProgramName(argv[0]);
   Py_Initialize();
   PySys_SetArgv(argc, argv);