From e2ccbccb02132cef59373bb8ec52ddbbf3c7c61d Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Wed, 9 Aug 2017 19:49:37 +0800 Subject: [PATCH 01/10] support python test without installation python package --- cmake/generic.cmake | 2 +- python/CMakeLists.txt | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/cmake/generic.cmake b/cmake/generic.cmake index 957c20bcf6..9f907a9dc2 100644 --- a/cmake/generic.cmake +++ b/cmake/generic.cmake @@ -411,7 +411,7 @@ function(py_test TARGET_NAME) set(multiValueArgs SRCS DEPS) cmake_parse_arguments(py_test "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) add_test(NAME ${TARGET_NAME} - COMMAND env PYTHONPATH=${PADDLE_PYTHON_PACKAGE_DIR} + COMMAND env PYTHONPATH=${PADDLE_PYTHON_LIB_DIR} python2 ${py_test_SRCS} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) endif() diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index b5030da8e7..fc8c6f6a42 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -1,6 +1,8 @@ set(OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/build") +set(PADDLE_PYTHON_LIB_DIR "${OUTPUT_DIR}/lib") + file(GLOB TRAINER_PY_FILES . ./paddle/trainer/*.py) file(GLOB HELPERS_PY_FILES . ./paddle/trainer_config_helpers/*.py) file(GLOB UTILS_PY_FILES . ./paddle/utils/*.py) From 5e5c441245276a2696ac1f840ebd261c7c14cfd4 Mon Sep 17 00:00:00 2001 From: liaogang Date: Wed, 9 Aug 2017 20:16:16 +0800 Subject: [PATCH 02/10] Enable Python Unit Test before make install --- cmake/generic.cmake | 2 +- paddle/framework/CMakeLists.txt | 5 +++++ python/paddle/v2/framework/.gitignore | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 python/paddle/v2/framework/.gitignore diff --git a/cmake/generic.cmake b/cmake/generic.cmake index 957c20bcf6..2778b49128 100644 --- a/cmake/generic.cmake +++ b/cmake/generic.cmake @@ -411,7 +411,7 @@ function(py_test TARGET_NAME) set(multiValueArgs SRCS DEPS) cmake_parse_arguments(py_test "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) add_test(NAME ${TARGET_NAME} - COMMAND env PYTHONPATH=${PADDLE_PYTHON_PACKAGE_DIR} + COMMAND env PYTHONPATH=${CMAKE_SOURCE_DIR}/python:${CMAKE_SOURCE_DIR}/paddle:${PADDLE_PYTHON_PACKAGE_DIR} python2 ${py_test_SRCS} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) endif() diff --git a/paddle/framework/CMakeLists.txt b/paddle/framework/CMakeLists.txt index 6601918c90..b7b61b597f 100644 --- a/paddle/framework/CMakeLists.txt +++ b/paddle/framework/CMakeLists.txt @@ -35,6 +35,11 @@ py_proto_compile(framework_py_proto SRCS attribute.proto op_proto.proto op_desc. # Generate an empty __init__.py to make framework_py_proto as a valid python module. add_custom_target(framework_py_proto_init ALL COMMAND ${CMAKE_COMMAND} -E touch __init__.py) add_dependencies(framework_py_proto framework_py_proto_init) +add_custom_command(TARGET framework_py_proto POST_BUILD + COMMAND ${CMAKE_COMMAND} -E make_directory ${PROJ_ROOT}/python/paddle/v2/framework/proto + COMMAND cp *.py ${PROJ_ROOT}/python/paddle/v2/framework/proto/ + COMMENT "Copy generated python proto into directory paddle/v2/framework/proto." + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) cc_library(backward SRCS backward.cc DEPS net_op) cc_test(backward_test SRCS backward_test.cc DEPS backward) diff --git a/python/paddle/v2/framework/.gitignore b/python/paddle/v2/framework/.gitignore new file mode 100644 index 0000000000..2ff540d576 --- /dev/null +++ b/python/paddle/v2/framework/.gitignore @@ -0,0 +1 @@ +proto From 8f464a58984f8024afadab2920acf2b9c4a60d17 Mon Sep 17 00:00:00 2001 From: liaogang Date: Wed, 9 Aug 2017 20:20:42 +0800 Subject: [PATCH 03/10] update PROJ_ROOT --- cmake/generic.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/generic.cmake b/cmake/generic.cmake index 2778b49128..6b0524021c 100644 --- a/cmake/generic.cmake +++ b/cmake/generic.cmake @@ -411,7 +411,7 @@ function(py_test TARGET_NAME) set(multiValueArgs SRCS DEPS) cmake_parse_arguments(py_test "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) add_test(NAME ${TARGET_NAME} - COMMAND env PYTHONPATH=${CMAKE_SOURCE_DIR}/python:${CMAKE_SOURCE_DIR}/paddle:${PADDLE_PYTHON_PACKAGE_DIR} + COMMAND env PYTHONPATH=${PROJ_ROOT}/python:${PROJ_ROOT}/paddle:${PADDLE_PYTHON_PACKAGE_DIR} python2 ${py_test_SRCS} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) endif() From aaddf5f6940768b827f03305e86da557ab24db65 Mon Sep 17 00:00:00 2001 From: liaogang Date: Thu, 10 Aug 2017 08:25:22 +0800 Subject: [PATCH 04/10] test on CI --- paddle/scripts/docker/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh index 44442be472..f70583c641 100644 --- a/paddle/scripts/docker/build.sh +++ b/paddle/scripts/docker/build.sh @@ -74,11 +74,11 @@ cat < Date: Thu, 10 Aug 2017 08:52:10 +0800 Subject: [PATCH 05/10] add python unit test dependencies --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 8ac123bf9c..c7fbe12c1b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -64,7 +64,7 @@ RUN pip install --upgrade pip && \ pip install -U sphinx-rtd-theme==0.1.9 recommonmark && \ pip install pre-commit 'requests==2.9.2' 'ipython==5.3.0' && \ pip install 'ipykernel==4.6.0' 'jupyter==1.0.0' && \ - pip install rarfile + pip install rarfile 'scipy>=0.19.0' 'nltk>=3.2.2' # To fix https://github.com/PaddlePaddle/Paddle/issues/1954, we use # the solution in https://urllib3.readthedocs.io/en/latest/user-guide.html#ssl-py2 From 6cd1617129620c88c84dcfe55f1e21e1882ef7e0 Mon Sep 17 00:00:00 2001 From: liaogang Date: Thu, 10 Aug 2017 09:20:16 +0800 Subject: [PATCH 06/10] add pip install opencv-python --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c7fbe12c1b..5d4de6e30d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -64,7 +64,7 @@ RUN pip install --upgrade pip && \ pip install -U sphinx-rtd-theme==0.1.9 recommonmark && \ pip install pre-commit 'requests==2.9.2' 'ipython==5.3.0' && \ pip install 'ipykernel==4.6.0' 'jupyter==1.0.0' && \ - pip install rarfile 'scipy>=0.19.0' 'nltk>=3.2.2' + pip install rarfile opencv-python 'scipy>=0.19.0' 'nltk>=3.2.2' # To fix https://github.com/PaddlePaddle/Paddle/issues/1954, we use # the solution in https://urllib3.readthedocs.io/en/latest/user-guide.html#ssl-py2 From 7d977e885ee22da42bd38731b90786fbc594f6eb Mon Sep 17 00:00:00 2001 From: liaogang Date: Thu, 10 Aug 2017 09:22:49 +0800 Subject: [PATCH 07/10] add pip install opencv-python --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c7fbe12c1b..5d4de6e30d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -64,7 +64,7 @@ RUN pip install --upgrade pip && \ pip install -U sphinx-rtd-theme==0.1.9 recommonmark && \ pip install pre-commit 'requests==2.9.2' 'ipython==5.3.0' && \ pip install 'ipykernel==4.6.0' 'jupyter==1.0.0' && \ - pip install rarfile 'scipy>=0.19.0' 'nltk>=3.2.2' + pip install rarfile opencv-python 'scipy>=0.19.0' 'nltk>=3.2.2' # To fix https://github.com/PaddlePaddle/Paddle/issues/1954, we use # the solution in https://urllib3.readthedocs.io/en/latest/user-guide.html#ssl-py2 From ca6c29635682ce1ebf4d42d7f9f0b94c2c88f6f4 Mon Sep 17 00:00:00 2001 From: liaogang Date: Thu, 10 Aug 2017 09:24:17 +0800 Subject: [PATCH 08/10] update --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 5d4de6e30d..0d0c88f40c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -64,7 +64,7 @@ RUN pip install --upgrade pip && \ pip install -U sphinx-rtd-theme==0.1.9 recommonmark && \ pip install pre-commit 'requests==2.9.2' 'ipython==5.3.0' && \ pip install 'ipykernel==4.6.0' 'jupyter==1.0.0' && \ - pip install rarfile opencv-python 'scipy>=0.19.0' 'nltk>=3.2.2' + pip install opencv-python rarfile 'scipy>=0.19.0' 'nltk>=3.2.2' # To fix https://github.com/PaddlePaddle/Paddle/issues/1954, we use # the solution in https://urllib3.readthedocs.io/en/latest/user-guide.html#ssl-py2 From c304e02813e0628acfbce0fb21239cca931483ca Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Thu, 10 Aug 2017 12:31:06 +0800 Subject: [PATCH 09/10] fix py_padde test --- CMakeLists.txt | 2 ++ cmake/generic.cmake | 2 +- python/CMakeLists.txt | 10 +++------- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b174831109..89e1fec566 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -164,10 +164,12 @@ if(WITH_GOLANG) add_subdirectory(go) endif(WITH_GOLANG) +set(PADDLE_PYTHON_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/python/build") add_subdirectory(paddle) if(WITH_PYTHON) add_subdirectory(python) endif() + if(WITH_DOC) add_subdirectory(doc) endif() diff --git a/cmake/generic.cmake b/cmake/generic.cmake index 9f907a9dc2..951642e70b 100644 --- a/cmake/generic.cmake +++ b/cmake/generic.cmake @@ -411,7 +411,7 @@ function(py_test TARGET_NAME) set(multiValueArgs SRCS DEPS) cmake_parse_arguments(py_test "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) add_test(NAME ${TARGET_NAME} - COMMAND env PYTHONPATH=${PADDLE_PYTHON_LIB_DIR} + COMMAND env PYTHONPATH=${PROJ_ROOT}/paddle:${PADDLE_PYTHON_BUILD_DIR}/lib python2 ${py_test_SRCS} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) endif() diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index fc8c6f6a42..684691d240 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -1,7 +1,3 @@ -set(OUTPUT_DIR - "${CMAKE_CURRENT_BINARY_DIR}/build") - -set(PADDLE_PYTHON_LIB_DIR "${OUTPUT_DIR}/lib") file(GLOB TRAINER_PY_FILES . ./paddle/trainer/*.py) file(GLOB HELPERS_PY_FILES . ./paddle/trainer_config_helpers/*.py) @@ -35,13 +31,13 @@ add_custom_command(OUTPUT ${PROJ_ROOT}/python/paddle/v2/framework/core.so add_custom_target(copy_paddle_pybind ALL DEPENDS ${PROJ_ROOT}/python/paddle/v2/framework/core.so) -add_custom_command(OUTPUT ${OUTPUT_DIR}/.timestamp +add_custom_command(OUTPUT ${PADDLE_PYTHON_BUILD_DIR}/.timestamp COMMAND env ${py_env} ${PYTHON_EXECUTABLE} setup.py bdist_wheel - COMMAND ${CMAKE_COMMAND} -E touch ${OUTPUT_DIR}/.timestamp + COMMAND ${CMAKE_COMMAND} -E touch ${PADDLE_PYTHON_BUILD_DIR}/.timestamp DEPENDS gen_proto_py copy_paddle_pybind framework_py_proto ${PY_FILES} ${external_project_dependencies} ${COPY_PADDLE_MASTER}) add_custom_target(paddle_python ALL DEPENDS - ${OUTPUT_DIR}/.timestamp paddle_pserver_main paddle_trainer paddle_merge_model python_api_wheel) + ${PADDLE_PYTHON_BUILD_DIR}/.timestamp paddle_pserver_main paddle_trainer paddle_merge_model python_api_wheel) set(PADDLE_PYTHON_PACKAGE_DIR ${CMAKE_CURRENT_BINARY_DIR}/dist/) From 3df8ee1ff778abf341b6391bec9b6a95001e004d Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Thu, 10 Aug 2017 18:22:41 +0800 Subject: [PATCH 10/10] use lib-python as python test dir --- cmake/generic.cmake | 2 +- python/CMakeLists.txt | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/cmake/generic.cmake b/cmake/generic.cmake index 951642e70b..d2aab938d4 100644 --- a/cmake/generic.cmake +++ b/cmake/generic.cmake @@ -411,7 +411,7 @@ function(py_test TARGET_NAME) set(multiValueArgs SRCS DEPS) cmake_parse_arguments(py_test "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) add_test(NAME ${TARGET_NAME} - COMMAND env PYTHONPATH=${PROJ_ROOT}/paddle:${PADDLE_PYTHON_BUILD_DIR}/lib + COMMAND env PYTHONPATH=${PADDLE_PYTHON_BUILD_DIR}/lib-python python2 ${py_test_SRCS} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) endif() diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 684691d240..0deff5ff08 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -34,6 +34,8 @@ add_custom_target(copy_paddle_pybind ALL DEPENDS ${PROJ_ROOT}/python/paddle/v2/f add_custom_command(OUTPUT ${PADDLE_PYTHON_BUILD_DIR}/.timestamp COMMAND env ${py_env} ${PYTHON_EXECUTABLE} setup.py bdist_wheel COMMAND ${CMAKE_COMMAND} -E touch ${PADDLE_PYTHON_BUILD_DIR}/.timestamp + COMMAND ${CMAKE_COMMAND} -E remove_directory ${PADDLE_PYTHON_BUILD_DIR}/lib-python + COMMAND ${CMAKE_COMMAND} -E copy_directory ${PADDLE_PYTHON_BUILD_DIR}/lib* ${PADDLE_PYTHON_BUILD_DIR}/lib-python DEPENDS gen_proto_py copy_paddle_pybind framework_py_proto ${PY_FILES} ${external_project_dependencies} ${COPY_PADDLE_MASTER}) add_custom_target(paddle_python ALL DEPENDS