From feccfc8333309e64623ec522d48284182c493117 Mon Sep 17 00:00:00 2001 From: qijun Date: Thu, 20 Apr 2017 14:27:02 +0800 Subject: [PATCH 01/13] fix with_doc build option --- paddle/scripts/docker/build.sh | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh index 16f669bd77..5156cf9147 100644 --- a/paddle/scripts/docker/build.sh +++ b/paddle/scripts/docker/build.sh @@ -29,7 +29,7 @@ rm *.deb 2>/dev/null || true cmake .. \ -DCMAKE_BUILD_TYPE=Release \ - -DWITH_DOC=${WITH_DOC:-OFF} \ + -DWITH_DOC=OFF \ -DWITH_GPU=${WITH_GPU:-OFF} \ -DWITH_AVX=${WITH_AVX:-OFF} \ -DWITH_SWIG_PY=ON \ @@ -100,6 +100,18 @@ RUN dpkg -i /usr/local/opt/paddle/deb/*.deb && \ rm -f /usr/local/opt/paddle/deb/*.deb && \ pip install /usr/opt/paddle/share/wheels/*.whl && \ paddle version +if [ ${WITH_DOC} == "ON" ]; then + mkdir -p /paddle/build_doc + pushd /paddle/build_doc + cmake .. \ + -DWITH_DOC=ON \ + -DWITH_GPU=OFF \ + -DWITH_AVX=${WITH_AVX:-OFF} \ + -DWITH_SWIG_PY=ON \ + -DWITH_STYLE_CHECK=OFF + make paddle_docs + popd +fi ${CPU_DOCKER_PYTHON_HOME_ENV} ${DOCKERFILE_CUDNN_DSO} ${DOCKERFILE_GPU_ENV} From 99ae4dae46a8cb4b2276f35619382ae51be06e3a Mon Sep 17 00:00:00 2001 From: qijun Date: Thu, 20 Apr 2017 14:30:40 +0800 Subject: [PATCH 02/13] add build paddle_docs_cn --- paddle/scripts/docker/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh index 5156cf9147..50b083fb39 100644 --- a/paddle/scripts/docker/build.sh +++ b/paddle/scripts/docker/build.sh @@ -109,7 +109,7 @@ if [ ${WITH_DOC} == "ON" ]; then -DWITH_AVX=${WITH_AVX:-OFF} \ -DWITH_SWIG_PY=ON \ -DWITH_STYLE_CHECK=OFF - make paddle_docs + make paddle_docs paddle_docs_cn popd fi ${CPU_DOCKER_PYTHON_HOME_ENV} From 4aabe597f62f19bd0c4478884d56a3db93740c36 Mon Sep 17 00:00:00 2001 From: qijun Date: Thu, 20 Apr 2017 14:38:10 +0800 Subject: [PATCH 03/13] fix dockerfile typo error --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 97947adf45..49dcc2ad96 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ ARG WITH_DOC ARG WITH_STYLE_CHECK ENV WOBOQ OFF -ENV WITH_GPU=${WITH_AVX:-OFF} +ENV WITH_GPU=${WITH_GPU:-OFF} ENV WITH_AVX=${WITH_AVX:-ON} ENV WITH_DOC=${WITH_DOC:-OFF} ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF} From 8dbbcc1fc11eb4403cabebe7577fa71f3508bd76 Mon Sep 17 00:00:00 2001 From: qijun Date: Thu, 20 Apr 2017 14:47:21 +0800 Subject: [PATCH 04/13] add install doxygen --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 49dcc2ad96..84f78ac5dd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,7 +27,7 @@ RUN apt-get update && \ apt-get install -y wget unzip tar xz-utils bzip2 gzip coreutils && \ apt-get install -y curl sed grep graphviz libjpeg-dev zlib1g-dev && \ apt-get install -y python-numpy python-matplotlib gcc g++ gfortran && \ - apt-get install -y automake locales clang-format-3.8 swig && \ + apt-get install -y automake locales clang-format-3.8 swig doxygen&& \ apt-get clean -y # git credential to skip password typing From 99567d4eb1eb8144cdbf5954eb03e32414867566 Mon Sep 17 00:00:00 2001 From: qijun Date: Thu, 20 Apr 2017 14:52:09 +0800 Subject: [PATCH 05/13] format code --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 84f78ac5dd..eaf472221a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,7 +27,7 @@ RUN apt-get update && \ apt-get install -y wget unzip tar xz-utils bzip2 gzip coreutils && \ apt-get install -y curl sed grep graphviz libjpeg-dev zlib1g-dev && \ apt-get install -y python-numpy python-matplotlib gcc g++ gfortran && \ - apt-get install -y automake locales clang-format-3.8 swig doxygen&& \ + apt-get install -y automake locales clang-format-3.8 swig doxygen && \ apt-get clean -y # git credential to skip password typing From f5d0970e13caeb097bdeb78378ab9ff733718208 Mon Sep 17 00:00:00 2001 From: qijun Date: Thu, 20 Apr 2017 17:08:36 +0800 Subject: [PATCH 06/13] fix ipython version --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index eaf472221a..f12be36ceb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -43,7 +43,8 @@ RUN pip install --upgrade pip && \ pip install -U wheel pillow BeautifulSoup && \ pip install -U docopt PyYAML sphinx && \ pip install -U sphinx-rtd-theme==0.1.9 recommonmark && \ - pip install pre-commit 'requests==2.9.2' 'ipykernel==4.6.0' 'jupyter==1.0.0' + pip install pre-commit 'requests==2.9.2' 'ipython==5.3.0' && \ + pip install 'ipykernel==4.6.0' 'jupyter==1.0.0' RUN curl -sSL https://cmake.org/files/v3.4/cmake-3.4.1.tar.gz | tar -xz && \ cd cmake-3.4.1 && ./bootstrap && make -j `nproc` && make install && \ From 0344c086e53a92822654741773ec839362c2a4f5 Mon Sep 17 00:00:00 2001 From: qijun Date: Thu, 20 Apr 2017 19:19:44 +0800 Subject: [PATCH 07/13] fix build.sh --- paddle/scripts/docker/build.sh | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh index 50b083fb39..c1e3f39888 100644 --- a/paddle/scripts/docker/build.sh +++ b/paddle/scripts/docker/build.sh @@ -42,7 +42,20 @@ if [[ ${RUN_TEST:-OFF} == "ON" ]]; then make coveralls fi make install +pip install /usr/opt/paddle/share/wheels/*.whl +if [ ${WITH_DOC} == "ON" ]; then + mkdir -p /paddle/build_doc + pushd /paddle/build_doc + cmake .. \ + -DWITH_DOC=ON \ + -DWITH_GPU=OFF \ + -DWITH_AVX=${WITH_AVX:-OFF} \ + -DWITH_SWIG_PY=ON \ + -DWITH_STYLE_CHECK=OFF + make paddle_docs paddle_docs_cn + popd +fi # generate deb package for current build # FIXME(typhoonzero): should we remove paddle/scripts/deb ? # FIXME: CPACK_DEBIAN_PACKAGE_DEPENDS removes all dev dependencies, must @@ -100,18 +113,6 @@ RUN dpkg -i /usr/local/opt/paddle/deb/*.deb && \ rm -f /usr/local/opt/paddle/deb/*.deb && \ pip install /usr/opt/paddle/share/wheels/*.whl && \ paddle version -if [ ${WITH_DOC} == "ON" ]; then - mkdir -p /paddle/build_doc - pushd /paddle/build_doc - cmake .. \ - -DWITH_DOC=ON \ - -DWITH_GPU=OFF \ - -DWITH_AVX=${WITH_AVX:-OFF} \ - -DWITH_SWIG_PY=ON \ - -DWITH_STYLE_CHECK=OFF - make paddle_docs paddle_docs_cn - popd -fi ${CPU_DOCKER_PYTHON_HOME_ENV} ${DOCKERFILE_CUDNN_DSO} ${DOCKERFILE_GPU_ENV} From 2925f0089a92f345a17725abd28e2cde02252f2e Mon Sep 17 00:00:00 2001 From: qijun Date: Thu, 20 Apr 2017 19:36:15 +0800 Subject: [PATCH 08/13] fix wrong path of pip install *whl --- paddle/scripts/docker/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh index c1e3f39888..6f7b97021d 100644 --- a/paddle/scripts/docker/build.sh +++ b/paddle/scripts/docker/build.sh @@ -42,7 +42,7 @@ if [[ ${RUN_TEST:-OFF} == "ON" ]]; then make coveralls fi make install -pip install /usr/opt/paddle/share/wheels/*.whl +pip install /usr/local/opt/paddle/share/wheels/*.whl if [ ${WITH_DOC} == "ON" ]; then mkdir -p /paddle/build_doc @@ -111,7 +111,7 @@ ADD build/*.deb /usr/local/opt/paddle/deb/ # run paddle version to install python packages first RUN dpkg -i /usr/local/opt/paddle/deb/*.deb && \ rm -f /usr/local/opt/paddle/deb/*.deb && \ - pip install /usr/opt/paddle/share/wheels/*.whl && \ + pip install /usr/local/opt/paddle/share/wheels/*.whl && \ paddle version ${CPU_DOCKER_PYTHON_HOME_ENV} ${DOCKERFILE_CUDNN_DSO} From 07d2e0f96fe569e73fd31ab9451cd3a1c4da9d39 Mon Sep 17 00:00:00 2001 From: qijun Date: Thu, 20 Apr 2017 19:48:54 +0800 Subject: [PATCH 09/13] add build_doc dir to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index ee7c6ec370..2b30f7938c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ *.DS_Store build/ +build_doc/ *.user .vscode From 81e0f5a4e25a20dcc198df26d8bc48e8df4c3f36 Mon Sep 17 00:00:00 2001 From: qijun Date: Fri, 21 Apr 2017 14:29:19 +0800 Subject: [PATCH 10/13] correct with_testing option --- paddle/scripts/docker/build.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh index 6f7b97021d..83cc30d3c2 100644 --- a/paddle/scripts/docker/build.sh +++ b/paddle/scripts/docker/build.sh @@ -35,11 +35,11 @@ cmake .. \ -DWITH_SWIG_PY=ON \ -DCUDNN_ROOT=/usr/ \ -DWITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF} \ - -DON_COVERALLS=${WITH_TEST:-OFF} \ + -DWITH_TESTING=${WITH_TESTING:-OFF} \ -DCMAKE_EXPORT_COMPILE_COMMANDS=ON make -j `nproc` -if [[ ${RUN_TEST:-OFF} == "ON" ]]; then - make coveralls +if [ ${WITH_TESTING:-OFF} == "ON" ] && [ ${RUN_TEST:-OFF} == "ON" ] ; then + make test fi make install pip install /usr/local/opt/paddle/share/wheels/*.whl From 190e429dfdac0c206b86bdd7bc515c06dfdb6390 Mon Sep 17 00:00:00 2001 From: qijun Date: Fri, 21 Apr 2017 14:35:05 +0800 Subject: [PATCH 11/13] add some comments --- paddle/scripts/docker/README.md | 2 +- paddle/scripts/docker/build.sh | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/paddle/scripts/docker/README.md b/paddle/scripts/docker/README.md index 132f8cd8aa..b45d92507c 100644 --- a/paddle/scripts/docker/README.md +++ b/paddle/scripts/docker/README.md @@ -94,7 +94,7 @@ docker build -t paddle:dev --build-arg UBUNTU_MIRROR=mirror://mirrors.ubuntu.com Given the development image `paddle:dev`, the following command builds PaddlePaddle from the source tree on the development computer (host): ```bash -docker run --rm -v $PWD:/paddle -e "WITH_GPU=OFF" -e "WITH_AVX=ON" -e "WITH_TEST=OFF" -e "RUN_TEST=OFF" paddle:dev +docker run --rm -v $PWD:/paddle -e "WITH_GPU=OFF" -e "WITH_AVX=ON" -e "WITH_TESTING=OFF" -e "RUN_TEST=OFF" paddle:dev ``` This command mounts the source directory on the host into `/paddle` in the container, so the default entry point of `paddle:dev`, `build.sh`, could build the source code with possible local changes. When it writes to `/paddle/build` in the container, it writes to `$PWD/build` on the host indeed. diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh index 83cc30d3c2..1613d970bc 100644 --- a/paddle/scripts/docker/build.sh +++ b/paddle/scripts/docker/build.sh @@ -44,6 +44,8 @@ fi make install pip install /usr/local/opt/paddle/share/wheels/*.whl +# Since python v2 api import py_paddle module, the generation of paddle docs +# depend on paddle's compilation and installation if [ ${WITH_DOC} == "ON" ]; then mkdir -p /paddle/build_doc pushd /paddle/build_doc From 2c4892c968560dcceec72f8cbc92d8ddc841aa2a Mon Sep 17 00:00:00 2001 From: qijun Date: Fri, 21 Apr 2017 18:56:11 +0800 Subject: [PATCH 12/13] find whl path --- paddle/scripts/docker/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh index 1613d970bc..185aa05077 100644 --- a/paddle/scripts/docker/build.sh +++ b/paddle/scripts/docker/build.sh @@ -113,7 +113,7 @@ ADD build/*.deb /usr/local/opt/paddle/deb/ # run paddle version to install python packages first RUN dpkg -i /usr/local/opt/paddle/deb/*.deb && \ rm -f /usr/local/opt/paddle/deb/*.deb && \ - pip install /usr/local/opt/paddle/share/wheels/*.whl && \ + find /usr/ -name '*paddle-*.whl' | xargs pip install && \ paddle version ${CPU_DOCKER_PYTHON_HOME_ENV} ${DOCKERFILE_CUDNN_DSO} From 7b16103f87332f07be2282937194d737fa397c14 Mon Sep 17 00:00:00 2001 From: qijun Date: Mon, 24 Apr 2017 10:33:56 +0800 Subject: [PATCH 13/13] update comments --- paddle/scripts/docker/build.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh index 185aa05077..73f885c1e7 100644 --- a/paddle/scripts/docker/build.sh +++ b/paddle/scripts/docker/build.sh @@ -44,8 +44,9 @@ fi make install pip install /usr/local/opt/paddle/share/wheels/*.whl -# Since python v2 api import py_paddle module, the generation of paddle docs -# depend on paddle's compilation and installation +# To build documentation, we need to run cmake twice. +# This awkwardness is due to https://github.com/PaddlePaddle/Paddle/issues/1854. +# It also describes a solution. if [ ${WITH_DOC} == "ON" ]; then mkdir -p /paddle/build_doc pushd /paddle/build_doc