From b098ef69a4a45efa8011a2f4b93ee55eca4bcc26 Mon Sep 17 00:00:00 2001 From: dzhwinter Date: Wed, 24 May 2017 19:26:39 +0800 Subject: [PATCH 1/9] "remove the rar extractfile, prevent small files" --- python/paddle/v2/dataset/__init__.py | 3 ++- python/paddle/v2/dataset/mq2007.py | 6 ++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/python/paddle/v2/dataset/__init__.py b/python/paddle/v2/dataset/__init__.py index 80ff6295c3..26252d5bbd 100644 --- a/python/paddle/v2/dataset/__init__.py +++ b/python/paddle/v2/dataset/__init__.py @@ -24,8 +24,9 @@ import conll05 import uci_housing import sentiment import wmt14 +import mq2007 __all__ = [ 'mnist', 'imikolov', 'imdb', 'cifar', 'movielens', 'conll05', 'sentiment' - 'uci_housing', 'wmt14' + 'uci_housing', 'wmt14', 'mq2007' ] diff --git a/python/paddle/v2/dataset/mq2007.py b/python/paddle/v2/dataset/mq2007.py index fd71b34166..d8c9918d14 100644 --- a/python/paddle/v2/dataset/mq2007.py +++ b/python/paddle/v2/dataset/mq2007.py @@ -41,9 +41,7 @@ def __initialize_meta_info__(): """ fn = fetch() rar = rarfile.RarFile(fn) - dirpath = os.path.dirname(fn) - rar.extractall(path=dirpath) - return dirpath + return rar class Query(object): @@ -273,7 +271,7 @@ def load_from_text(filepath, shuffle=True, fill_missing=-1): querylists = [] querylist = None fn = __initialize_meta_info__() - with open(os.path.join(fn, filepath)) as f: + with fn.open(os.path.join(fn, filepath)) as f: for line in f: query = Query() query = query._parse_(line) From 6e91ebc0dd9dbb2ce9142216a1aa0c4575f350eb Mon Sep 17 00:00:00 2001 From: dzhwinter Date: Wed, 24 May 2017 19:30:27 +0800 Subject: [PATCH 2/9] "remove extrafile" --- python/paddle/v2/dataset/mq2007.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/python/paddle/v2/dataset/mq2007.py b/python/paddle/v2/dataset/mq2007.py index d8c9918d14..fd71b34166 100644 --- a/python/paddle/v2/dataset/mq2007.py +++ b/python/paddle/v2/dataset/mq2007.py @@ -41,7 +41,9 @@ def __initialize_meta_info__(): """ fn = fetch() rar = rarfile.RarFile(fn) - return rar + dirpath = os.path.dirname(fn) + rar.extractall(path=dirpath) + return dirpath class Query(object): @@ -271,7 +273,7 @@ def load_from_text(filepath, shuffle=True, fill_missing=-1): querylists = [] querylist = None fn = __initialize_meta_info__() - with fn.open(os.path.join(fn, filepath)) as f: + with open(os.path.join(fn, filepath)) as f: for line in f: query = Query() query = query._parse_(line) From f9eb4b74b2d572f1237d7fdecb5ac7f4bc02f22f Mon Sep 17 00:00:00 2001 From: dzhwinter Date: Wed, 24 May 2017 20:22:16 +0800 Subject: [PATCH 3/9] "update python package" --- .travis.yml | 2 +- Dockerfile | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 387367a230..2dad0e77e6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,7 +48,7 @@ before_install: - if [[ "$JOB" == "PRE_COMMIT" ]]; then sudo ln -s /usr/bin/clang-format-3.8 /usr/bin/clang-format; fi # Paddle is using protobuf 3.1 currently. Protobuf 3.2 breaks the compatibility. So we specify the python # protobuf version. - - pip install numpy wheel 'protobuf==3.1' sphinx==1.5.6 recommonmark sphinx-rtd-theme==0.1.9 virtualenv pre-commit requests==2.9.2 LinkChecker + - pip install numpy wheel 'protobuf==3.1' sphinx==1.5.6 recommonmark sphinx-rtd-theme==0.1.9 virtualenv pre-commit requests==2.9.2 LinkChecker rarfile - | function timeout() { perl -e 'alarm shift; exec @ARGV' "$@"; } script: diff --git a/Dockerfile b/Dockerfile index 571c3e1476..b6f99ca539 100644 --- a/Dockerfile +++ b/Dockerfile @@ -56,7 +56,8 @@ RUN pip install --upgrade pip && \ pip install -U docopt PyYAML sphinx && \ pip install -U sphinx-rtd-theme==0.1.9 recommonmark && \ pip install pre-commit 'requests==2.9.2' 'ipython==5.3.0' && \ - pip install 'ipykernel==4.6.0' 'jupyter==1.0.0' + pip install 'ipykernel==4.6.0' 'jupyter==1.0.0' && \ + pip install rarfile # To fix https://github.com/PaddlePaddle/Paddle/issues/1954, we use # the solution in https://urllib3.readthedocs.io/en/latest/user-guide.html#ssl-py2 From d3bf8ca161a73f3355f38c8ea8f6355cde1c5349 Mon Sep 17 00:00:00 2001 From: dzhwinter Date: Wed, 24 May 2017 22:00:45 +0800 Subject: [PATCH 4/9] "fix travis failed on rarfile " --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 2dad0e77e6..44b755ee32 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,7 +48,8 @@ before_install: - if [[ "$JOB" == "PRE_COMMIT" ]]; then sudo ln -s /usr/bin/clang-format-3.8 /usr/bin/clang-format; fi # Paddle is using protobuf 3.1 currently. Protobuf 3.2 breaks the compatibility. So we specify the python # protobuf version. - - pip install numpy wheel 'protobuf==3.1' sphinx==1.5.6 recommonmark sphinx-rtd-theme==0.1.9 virtualenv pre-commit requests==2.9.2 LinkChecker rarfile + - pip install numpy wheel 'protobuf==3.1' sphinx==1.5.6 recommonmark sphinx-rtd-theme==0.1.9 virtualenv pre-commit requests==2.9.2 LinkChecker + - pip install rarfile - | function timeout() { perl -e 'alarm shift; exec @ARGV' "$@"; } script: From 8ecd722958a182e923a5476cbb5a9dff0ce7deb8 Mon Sep 17 00:00:00 2001 From: dzhwinter Date: Wed, 24 May 2017 23:06:33 +0800 Subject: [PATCH 5/9] "update setup.in" --- python/setup.py.in | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/setup.py.in b/python/setup.py.in index 5dfb46192a..1afaffd261 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -1,5 +1,6 @@ from setuptools import setup + packages=['paddle', 'paddle.proto', 'paddle.trainer', @@ -18,6 +19,7 @@ setup(name='paddle', "numpy", "protobuf==${PROTOBUF_VERSION}", "matplotlib", + "rarfile" ], packages=packages, package_dir={ From 8813ad9d2f4bfed97e2e24a5e20fa4b6db561a50 Mon Sep 17 00:00:00 2001 From: dzhwinter Date: Thu, 25 May 2017 11:26:38 +0800 Subject: [PATCH 6/9] "fix dependency build failed" --- paddle/scripts/run_python_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/scripts/run_python_tests.sh b/paddle/scripts/run_python_tests.sh index 02d2cdb977..c588b9e08d 100755 --- a/paddle/scripts/run_python_tests.sh +++ b/paddle/scripts/run_python_tests.sh @@ -29,7 +29,7 @@ if [ $USE_VIRTUALENV_FOR_TEST -ne 0 ]; then fi export PYTHONPATH=$SCRIPTPATH/../../python/ -$PYTHON -m pip install $SCRIPTPATH/../dist/*.whl requests matplotlib opencv-python ipython==5.3 +$PYTHON -m pip install $SCRIPTPATH/../dist/*.whl requests matplotlib opencv-python ipython==5.3 rarfile for fn in "$@" do From f170111031b1f202effea678ebb5eaa4a9bf13bb Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Thu, 25 May 2017 12:18:16 +0800 Subject: [PATCH 7/9] Fix compile when not enable testing * WITH_TESTING is a flag that user could configure --- paddle/go/CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/paddle/go/CMakeLists.txt b/paddle/go/CMakeLists.txt index 20f1476943..51c5252d66 100644 --- a/paddle/go/CMakeLists.txt +++ b/paddle/go/CMakeLists.txt @@ -2,8 +2,10 @@ include_directories(${CMAKE_CURRENT_BINARY_DIR}) go_library(adder SRCS adder.go) -cc_test(cgo_test +if (WITH_TESTING) + cc_test(cgo_test SRCS cgo_test.cc DEPS adder) +endif() From 19127b479c40d5f00c4310eb44e2888f52c65978 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Thu, 25 May 2017 12:16:56 +0800 Subject: [PATCH 8/9] Fix bug in run_python_tests.sh * Also, test_plot cannot run in MacOS because of matplotlib & ipython's limit. * Add missing dependency in setup.py. * fix #2264 --- cmake/util.cmake | 5 +++-- paddle/scripts/run_python_tests.sh | 9 +++++++-- python/CMakeLists.txt | 4 +++- python/paddle/v2/plot/tests/CMakeLists.txt | 6 +++++- python/setup.py.in | 1 + 5 files changed, 19 insertions(+), 6 deletions(-) diff --git a/cmake/util.cmake b/cmake/util.cmake index b828eef322..8c91434622 100644 --- a/cmake/util.cmake +++ b/cmake/util.cmake @@ -149,8 +149,9 @@ endfunction() # Create a python unittest using run_python_tests.sh, # which takes care of making correct running environment function(add_python_test TEST_NAME) - add_test(NAME ${TEST_NAME} - COMMAND bash ${PROJ_ROOT}/paddle/scripts/run_python_tests.sh + add_test(NAME ${TEST_NAME} + COMMAND env PADDLE_PACKAGE_DIR=${PADDLE_PYTHON_PACKAGE_DIR} + bash ${PROJ_ROOT}/paddle/scripts/run_python_tests.sh ${USE_VIRTUALENV_FOR_TEST} ${PYTHON_EXECUTABLE} ${ARGN} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) endfunction() diff --git a/paddle/scripts/run_python_tests.sh b/paddle/scripts/run_python_tests.sh index 02d2cdb977..6dfc05a949 100755 --- a/paddle/scripts/run_python_tests.sh +++ b/paddle/scripts/run_python_tests.sh @@ -28,8 +28,13 @@ if [ $USE_VIRTUALENV_FOR_TEST -ne 0 ]; then PYTHON=python fi -export PYTHONPATH=$SCRIPTPATH/../../python/ -$PYTHON -m pip install $SCRIPTPATH/../dist/*.whl requests matplotlib opencv-python ipython==5.3 +$PYTHON -m pip install $SCRIPTPATH/../dist/*.whl + +if [ "X${PADDLE_PACKAGE_DIR}" != "X" ]; then + $PYTHON -m pip install ${PADDLE_PACKAGE_DIR}/*.whl +fi + +$PYTHON -m pip install ipython==5.3 for fn in "$@" do diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index bfa19d5ecc..4f52f0f6cf 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -23,7 +23,9 @@ add_custom_command(OUTPUT ${OUTPUT_DIR}/.timestamp add_custom_target(paddle_python ALL DEPENDS ${OUTPUT_DIR}/.timestamp) +set(PADDLE_PYTHON_PACKAGE_DIR ${CMAKE_CURRENT_BINARY_DIR}/dist/) add_subdirectory(paddle/trainer_config_helpers/tests) + if (WITH_SWIG_PY) # enable v2 API unittest only when paddle swig api is compiled add_subdirectory(paddle/v2/tests) @@ -31,6 +33,6 @@ if (WITH_SWIG_PY) add_subdirectory(paddle/v2/plot/tests) endif() -install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/dist/ +install(DIRECTORY ${PADDLE_PYTHON_PACKAGE_DIR} DESTINATION opt/paddle/share/wheels ) diff --git a/python/paddle/v2/plot/tests/CMakeLists.txt b/python/paddle/v2/plot/tests/CMakeLists.txt index b1132f1317..da5cd76488 100644 --- a/python/paddle/v2/plot/tests/CMakeLists.txt +++ b/python/paddle/v2/plot/tests/CMakeLists.txt @@ -1 +1,5 @@ -add_python_test(test_ploter test_ploter.py) +if (NOT APPLE) + # The Mac OS X backend will not be able to function correctly if Python is + # not installed as a framework. + add_python_test(test_ploter test_ploter.py) +endif() diff --git a/python/setup.py.in b/python/setup.py.in index 5dfb46192a..d747288434 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -18,6 +18,7 @@ setup(name='paddle', "numpy", "protobuf==${PROTOBUF_VERSION}", "matplotlib", + "opencv-python" ], packages=packages, package_dir={ From ec6068b187c679727246ebbed43468d56e274cb6 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Thu, 25 May 2017 13:45:37 +0800 Subject: [PATCH 9/9] Tinker with TravisCI --- paddle/scripts/run_python_tests.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/paddle/scripts/run_python_tests.sh b/paddle/scripts/run_python_tests.sh index 6dfc05a949..1ed497aaec 100755 --- a/paddle/scripts/run_python_tests.sh +++ b/paddle/scripts/run_python_tests.sh @@ -24,6 +24,8 @@ PYTHON=$1; shift if [ $USE_VIRTUALENV_FOR_TEST -ne 0 ]; then rm -rf .test_env virtualenv .test_env + unset PYTHONHOME + unset PYTHONPATH source .test_env/bin/activate PYTHON=python fi @@ -32,6 +34,8 @@ $PYTHON -m pip install $SCRIPTPATH/../dist/*.whl if [ "X${PADDLE_PACKAGE_DIR}" != "X" ]; then $PYTHON -m pip install ${PADDLE_PACKAGE_DIR}/*.whl +else + export PYTHONPATH=$SCRIPTPATH/../../python/ fi $PYTHON -m pip install ipython==5.3