From 359e79e5431fe7aefacf81a2e0c663b5074590de Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Thu, 3 Aug 2017 21:24:44 +0800 Subject: [PATCH 1/8] Faster Python unittest * Do not use `virtualenv` in Docker * Split unittest into many target, make them parallel --- CMakeLists.txt | 1 + cmake/external/python.cmake | 1 - cmake/util.cmake | 20 +++++++++++++++----- paddle/scripts/docker/build.sh | 26 ++++++++++++-------------- paddle/scripts/run_python_tests.sh | 23 ++++++++--------------- 5 files changed, 36 insertions(+), 35 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b174831109..236363944e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -55,6 +55,7 @@ option(WITH_C_API "Compile PaddlePaddle with C-API(Prediction)" OFF) option(WITH_GOLANG "Compile PaddlePaddle with GOLANG" OFF) option(GLIDE_INSTALL "Download and install go dependencies " ON) option(USE_NNPACK "Compile PaddlePaddle with NNPACK library" OFF) +option(UNITTEST_USE_VIRTUALENV "Python unittest with virtualenv" ON) # CMAKE_BUILD_TYPE if(NOT CMAKE_BUILD_TYPE) diff --git a/cmake/external/python.cmake b/cmake/external/python.cmake index 67a359d4b5..490c87d67e 100644 --- a/cmake/external/python.cmake +++ b/cmake/external/python.cmake @@ -24,7 +24,6 @@ IF(WITH_PYTHON) ENDIF(WITH_PYTHON) SET(py_env "") -SET(USE_VIRTUALENV_FOR_TEST 1) IF(PYTHONINTERP_FOUND) find_python_module(pip REQUIRED) find_python_module(numpy REQUIRED) diff --git a/cmake/util.cmake b/cmake/util.cmake index 87ad9d91d8..9790016df9 100644 --- a/cmake/util.cmake +++ b/cmake/util.cmake @@ -150,9 +150,19 @@ endfunction() # Create a python unittest using run_python_tests.sh, # which takes care of making correct running environment function(add_python_test TEST_NAME) - add_test(NAME ${TEST_NAME} - COMMAND env PADDLE_PACKAGE_DIR=${PADDLE_PYTHON_PACKAGE_DIR} - bash ${PROJ_ROOT}/paddle/scripts/run_python_tests.sh - ${USE_VIRTUALENV_FOR_TEST} ${PYTHON_EXECUTABLE} ${ARGN} - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) + if (UNITTEST_USE_VIRTUALENV) + add_test(NAME ${TEST_NAME} + COMMAND env PADDLE_PACKAGE_DIR=${PADDLE_PYTHON_PACKAGE_DIR} + bash ${PROJ_ROOT}/paddle/scripts/run_python_tests.sh ${ARGN} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) + else() + foreach(arg ${ARGN}) + get_filename_component(py_fn ${arg} NAME_WE) + set(TRG_NAME ${TEST_NAME}_${py_fn}) + add_test(NAME ${TRG_NAME} + COMMAND env PYTHONPATH=${PADDLE_PYTHON_PACKAGE_DIR} + python2 ${arg} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) + endforeach() + endif() endfunction() diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh index 8de0e608c1..f50b793bf5 100644 --- a/paddle/scripts/docker/build.sh +++ b/paddle/scripts/docker/build.sh @@ -49,29 +49,27 @@ cmake .. \ -DCUDNN_ROOT=/usr/ \ -DWITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF} \ -DWITH_TESTING=${WITH_TESTING:-OFF} \ - -DCMAKE_EXPORT_COMPILE_COMMANDS=ON + -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ + -DUNITTEST_USE_VIRTUALENV=OFF cat < /dev/null SCRIPTPATH=$PWD popd > /dev/null -USE_VIRTUALENV_FOR_TEST=$1; shift -PYTHON=$1; shift - -if [ $USE_VIRTUALENV_FOR_TEST -ne 0 ]; then - rm -rf .test_env - virtualenv .test_env - unset PYTHONHOME - unset PYTHONPATH - source .test_env/bin/activate - PYTHON=python -fi +rm -rf .test_env +virtualenv .test_env +unset PYTHONHOME +unset PYTHONPATH +source .test_env/bin/activate +PYTHON=python $PYTHON -m pip install $SCRIPTPATH/../dist/*.whl @@ -49,7 +44,5 @@ do fi done -if [ $USE_VIRTUALENV_FOR_TEST -ne 0 ]; then - deactivate - rm -rf .test_env -fi +deactivate +rm -rf .test_env From 52b68cad4270ae35cdd8a82b4129f955beb62c5d Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Fri, 4 Aug 2017 12:23:34 +0800 Subject: [PATCH 2/8] Update set python path --- paddle/.set_python_path.sh | 31 ++++++++++++------------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/paddle/.set_python_path.sh b/paddle/.set_python_path.sh index fa7baccc86..8fd58925ee 100755 --- a/paddle/.set_python_path.sh +++ b/paddle/.set_python_path.sh @@ -21,22 +21,15 @@ # # It same as PYTHONPATH=${YOUR_PYTHON_PATH}:$PYTHONPATH {exec...} # - -if ! python -c "import paddle" >/dev/null 2>/dev/null; then - PYPATH="" - set -x - while getopts "d:" opt; do - case $opt in - d) - PYPATH=$OPTARG - ;; - esac - done - shift $(($OPTIND - 1)) - export PYTHONPATH=$PYPATH:$PYTHONPATH - $@ -else - echo "paddle package is already in your PYTHONPATH. But unittest need a clean environment." - echo "Please uninstall paddle package before start unittest. Try to 'pip uninstall paddle'" - exit 1 -fi +PYPATH="" +set -x +while getopts "d:" opt; do + case $opt in + d) + PYPATH=$OPTARG + ;; + esac +done +shift $(($OPTIND - 1)) +export PYTHONPATH=$PYPATH:$PYTHONPATH +$@ From 36d40b1b70b177e3696f1a96db6811dd5ab319d1 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Fri, 4 Aug 2017 13:38:12 +0800 Subject: [PATCH 3/8] Paddle needs numpy>=1.12 * because it uses `numpy.flip` in `test_image.py` --- paddle/setup.py.in | 2 +- python/setup.py.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/paddle/setup.py.in b/paddle/setup.py.in index 06d55d3abc..31c9329f6a 100644 --- a/paddle/setup.py.in +++ b/paddle/setup.py.in @@ -22,7 +22,7 @@ setup(name="py_paddle", package_data={'py_paddle':['*.py','_swig_paddle.so']}, install_requires = [ 'nltk>=3.2.2', - 'numpy>=1.8.0', # The numpy is required. + 'numpy>=1.12.0', # The numpy is required. 'protobuf==${PROTOBUF_VERSION}' # The paddle protobuf version ], url='http://www.paddlepaddle.org/', diff --git a/python/setup.py.in b/python/setup.py.in index 65a26940d4..7808238aa6 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -14,7 +14,7 @@ packages=['paddle', 'paddle.v2.framework.proto'] setup_requires=["requests", - "numpy", + "numpy>=1.12", "protobuf==3.1", "recordio", "matplotlib", From 6cde6fc1873befb3502986b81489f9c0e1650c1e Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Fri, 4 Aug 2017 15:30:01 +0800 Subject: [PATCH 4/8] Follow comments --- paddle/scripts/docker/build.sh | 5 +++++ paddle/setup.py.in | 2 ++ 2 files changed, 7 insertions(+) diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh index f50b793bf5..147c0f3e64 100644 --- a/paddle/scripts/docker/build.sh +++ b/paddle/scripts/docker/build.sh @@ -37,8 +37,13 @@ Configuring cmake in /paddle/build ... -DWITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF} -DWITH_TESTING=${WITH_TESTING:-OFF} -DCMAKE_EXPORT_COMPILE_COMMANDS=ON + -DUNITTEST_USE_VIRTUALENV=OFF ======================================== EOF + +# Disable UNITTEST_USE_VIRTUALENV in docker because +# docker environment is fully controlled by this script. +# See /Paddle/CMakeLists.txt, UNITTEST_USE_VIRTUALENV option. cmake .. \ -DCMAKE_BUILD_TYPE=Release \ -DWITH_DOC=OFF \ diff --git a/paddle/setup.py.in b/paddle/setup.py.in index 31c9329f6a..af107e7672 100644 --- a/paddle/setup.py.in +++ b/paddle/setup.py.in @@ -22,6 +22,8 @@ setup(name="py_paddle", package_data={'py_paddle':['*.py','_swig_paddle.so']}, install_requires = [ 'nltk>=3.2.2', + # We use `numpy.flip` in `test_image.py`. + # `numpy.flip` is introduced in `1.12.0` 'numpy>=1.12.0', # The numpy is required. 'protobuf==${PROTOBUF_VERSION}' # The paddle protobuf version ], From aa57f0fc853f18651c0956a36fcf309f204e39b4 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Fri, 4 Aug 2017 16:23:04 +0800 Subject: [PATCH 5/8] Cat error log if cpack failed --- paddle/scripts/docker/build.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh index 147c0f3e64..7b17363226 100644 --- a/paddle/scripts/docker/build.sh +++ b/paddle/scripts/docker/build.sh @@ -125,9 +125,15 @@ cat < Date: Sat, 5 Aug 2017 12:12:33 +0800 Subject: [PATCH 6/8] Update dockerfile's numpy --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 5dd9b0be4f..d9e730cadb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,12 +27,13 @@ RUN apt-get update && \ git python-pip python-dev openssh-server bison \ wget unzip unrar tar xz-utils bzip2 gzip coreutils ntp \ curl sed grep graphviz libjpeg-dev zlib1g-dev \ - python-numpy python-matplotlib gcc-4.8 g++-4.8 \ + python-matplotlib gcc-4.8 g++-4.8 \ automake locales clang-format-3.8 swig doxygen cmake \ liblapack-dev liblapacke-dev libboost-dev \ clang-3.8 llvm-3.8 libclang-3.8-dev \ net-tools && \ apt-get clean -y +RUN pip install -y 'numpy>=1.12.0' # Install Go and glide RUN wget -O go.tgz https://storage.googleapis.com/golang/go1.8.1.linux-amd64.tar.gz && \ From 49b9f6eb45ecfb34165a8248ea6e9eeec5495a6d Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Sat, 5 Aug 2017 12:13:25 +0800 Subject: [PATCH 7/8] Update numpy version in docker file --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index d9e730cadb..b4a427cc4d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,6 +33,8 @@ RUN apt-get update && \ clang-3.8 llvm-3.8 libclang-3.8-dev \ net-tools && \ apt-get clean -y + +# paddle is using numpy.flip, which is introduced since 1.12.0 RUN pip install -y 'numpy>=1.12.0' # Install Go and glide From a720d21dcd60c25f20342d6256c54f2927154c2d Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Sat, 5 Aug 2017 16:11:43 +0800 Subject: [PATCH 8/8] Paddle always not use `virtualenv` to unit-test * User must install Paddle python package before unittest. * Or use docker to build Paddle --- CMakeLists.txt | 1 - Dockerfile | 2 +- cmake/util.cmake | 23 +++++--------- paddle/scripts/docker/build.sh | 4 +-- paddle/scripts/run_python_tests.sh | 48 ------------------------------ 5 files changed, 10 insertions(+), 68 deletions(-) delete mode 100755 paddle/scripts/run_python_tests.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 59407f638f..c7d743e193 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -55,7 +55,6 @@ option(WITH_C_API "Compile PaddlePaddle with C-API(Prediction)" OFF) option(WITH_GOLANG "Compile PaddlePaddle with GOLANG" OFF) option(GLIDE_INSTALL "Download and install go dependencies " ON) option(USE_NNPACK "Compile PaddlePaddle with NNPACK library" OFF) -option(UNITTEST_USE_VIRTUALENV "Python unittest with virtualenv" ON) # CMAKE_BUILD_TYPE if(NOT CMAKE_BUILD_TYPE) diff --git a/Dockerfile b/Dockerfile index b4a427cc4d..156ad3552b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,7 +35,7 @@ RUN apt-get update && \ apt-get clean -y # paddle is using numpy.flip, which is introduced since 1.12.0 -RUN pip install -y 'numpy>=1.12.0' +RUN pip --no-cache-dir install 'numpy>=1.12.0' # Install Go and glide RUN wget -O go.tgz https://storage.googleapis.com/golang/go1.8.1.linux-amd64.tar.gz && \ diff --git a/cmake/util.cmake b/cmake/util.cmake index 3391527e5a..4a27623b7f 100644 --- a/cmake/util.cmake +++ b/cmake/util.cmake @@ -149,19 +149,12 @@ endfunction() # Create a python unittest using run_python_tests.sh, # which takes care of making correct running environment function(add_python_test TEST_NAME) - if (UNITTEST_USE_VIRTUALENV) - add_test(NAME ${TEST_NAME} - COMMAND env PADDLE_PACKAGE_DIR=${PADDLE_PYTHON_PACKAGE_DIR} - bash ${PROJ_ROOT}/paddle/scripts/run_python_tests.sh ${ARGN} - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) - else() - foreach(arg ${ARGN}) - get_filename_component(py_fn ${arg} NAME_WE) - set(TRG_NAME ${TEST_NAME}_${py_fn}) - add_test(NAME ${TRG_NAME} - COMMAND env PYTHONPATH=${PADDLE_PYTHON_PACKAGE_DIR} - python2 ${arg} - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) - endforeach() - endif() + foreach(arg ${ARGN}) + get_filename_component(py_fn ${arg} NAME_WE) + set(TRG_NAME ${TEST_NAME}_${py_fn}) + add_test(NAME ${TRG_NAME} + COMMAND env PYTHONPATH=${PADDLE_PYTHON_PACKAGE_DIR} + python2 ${arg} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) + endforeach() endfunction() diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh index 7b17363226..92e59a27ac 100644 --- a/paddle/scripts/docker/build.sh +++ b/paddle/scripts/docker/build.sh @@ -37,7 +37,6 @@ Configuring cmake in /paddle/build ... -DWITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF} -DWITH_TESTING=${WITH_TESTING:-OFF} -DCMAKE_EXPORT_COMPILE_COMMANDS=ON - -DUNITTEST_USE_VIRTUALENV=OFF ======================================== EOF @@ -54,8 +53,7 @@ cmake .. \ -DCUDNN_ROOT=/usr/ \ -DWITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF} \ -DWITH_TESTING=${WITH_TESTING:-OFF} \ - -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ - -DUNITTEST_USE_VIRTUALENV=OFF + -DCMAKE_EXPORT_COMPILE_COMMANDS=ON cat < /dev/null -SCRIPTPATH=$PWD -popd > /dev/null - -rm -rf .test_env -virtualenv .test_env -unset PYTHONHOME -unset PYTHONPATH -source .test_env/bin/activate -PYTHON=python - -$PYTHON -m pip install $SCRIPTPATH/../dist/*.whl - -if [ "X${PADDLE_PACKAGE_DIR}" != "X" ]; then - $PYTHON -m pip install ${PADDLE_PACKAGE_DIR}/*.whl -else - export PYTHONPATH=$SCRIPTPATH/../../python/ -fi - -$PYTHON -m pip install ipython==5.3 - -for fn in "$@" -do - echo "test $fn" - $PYTHON $fn - if [ $? -ne 0 ]; then - exit 1 - fi -done - -deactivate -rm -rf .test_env