From 4d3f9107c7b7a424478e59c6f29a75ef78d68b50 Mon Sep 17 00:00:00 2001 From: liaogang Date: Thu, 20 Apr 2017 16:00:09 +0800 Subject: [PATCH 01/19] Add pip install paddle wheel package in Dockerfile --- paddle/scripts/docker/build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh index a4b63f90ec..aee3ffc6b9 100644 --- a/paddle/scripts/docker/build.sh +++ b/paddle/scripts/docker/build.sh @@ -99,6 +99,7 @@ ADD build/*.deb /usr/local/opt/paddle/deb/ # run paddle version to install python packages first RUN dpkg -i /usr/local/opt/paddle/deb/*.deb && \ rm -f /usr/local/opt/paddle/deb/*.deb && \ + pip install /usr/opt/paddle/share/wheels/*.whl && \ paddle version ${CPU_DOCKER_PYTHON_HOME_ENV} ${DOCKERFILE_CUDNN_DSO} From 771bbe60276e6d7e7cc9488be6d035972cd1a7f6 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Wed, 19 Apr 2017 20:29:43 +0800 Subject: [PATCH 02/19] Fix Production Docker Image's mirror settings. --- paddle/scripts/docker/build.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh index aee3ffc6b9..16f669bd77 100644 --- a/paddle/scripts/docker/build.sh +++ b/paddle/scripts/docker/build.sh @@ -75,9 +75,8 @@ fi paddle version -# generate production docker image Dockerfile -if [ ${USE_MIRROR} ]; then - MIRROR_UPDATE="sed 's@http:\/\/archive.ubuntu.com\/ubuntu\/@mirror:\/\/mirrors.ubuntu.com\/mirrors.txt@' -i /etc/apt/sources.list && \\" +if [[ -n ${APT_MIRROR} ]]; then + MIRROR_UPDATE="sed -i '${APT_MIRROR}' /etc/apt/sources.list && \\" else MIRROR_UPDATE="\\" fi From fe456676550cbf44e838d323a864649ae2fca6fa Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Fri, 21 Apr 2017 00:12:22 +0800 Subject: [PATCH 03/19] Timeout for caching in TravisCI --- .travis.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5a7f45a748..865e21f046 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,6 @@ env: - JOB=DOCS - JOB=BUILD_AND_TEST - JOB=PRE_COMMIT - addons: apt: packages: @@ -49,8 +48,12 @@ before_install: # Paddle is using protobuf 3.1 currently. Protobuf 3.2 breaks the compatibility. So we specify the python # protobuf version. - pip install numpy wheel 'protobuf==3.1' sphinx recommonmark sphinx-rtd-theme==0.1.9 virtualenv pre-commit requests==2.9.2 LinkChecker + - | + function timeout() { perl -e 'alarm shift; exec @ARGV' "$@"; } script: - - paddle/scripts/travis/main.sh + - | + timeout 2580 paddle/scripts/travis/main.sh # 43min timeout + RESULT=$?; if [ $RESULT -eq 0 ] || [ $RESULT -eq 142 ]; then true; else false; fi; notifications: email: on_success: change From 07de6022212ffda1dbf73dc62bb753f76697bfb8 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Fri, 21 Apr 2017 19:14:12 +0800 Subject: [PATCH 04/19] Correct model average settings in crf --- demo/sequence_tagging/linear_crf.py | 2 +- demo/sequence_tagging/rnn_crf.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/demo/sequence_tagging/linear_crf.py b/demo/sequence_tagging/linear_crf.py index 0624b17787..ea012ba1ae 100644 --- a/demo/sequence_tagging/linear_crf.py +++ b/demo/sequence_tagging/linear_crf.py @@ -27,7 +27,7 @@ settings( learning_method=MomentumOptimizer(), batch_size=batch_size, regularization=L2Regularization(batch_size * 1e-4), - average_window=0.5, + model_average=ModelAverage(0.5), learning_rate=1e-1, learning_rate_decay_a=1e-5, learning_rate_decay_b=0.25, ) diff --git a/demo/sequence_tagging/rnn_crf.py b/demo/sequence_tagging/rnn_crf.py index b9b41b2433..937a34df10 100644 --- a/demo/sequence_tagging/rnn_crf.py +++ b/demo/sequence_tagging/rnn_crf.py @@ -27,7 +27,7 @@ settings( learning_method=MomentumOptimizer(), batch_size=batch_size, regularization=L2Regularization(batch_size * 1e-5), - average_window=0.5, + model_average=ModelAverage(0.5), learning_rate=2e-3, learning_rate_decay_a=5e-7, learning_rate_decay_b=0.5, ) From 8b5e0275b29021117f6db2793779d6f5fb2a8c28 Mon Sep 17 00:00:00 2001 From: Luo Tao Date: Fri, 21 Apr 2017 17:59:33 +0800 Subject: [PATCH 05/19] fix sphinx doc bug --- doc/api/v2/config/layer.rst | 215 ++++++------------ .../paddle/trainer_config_helpers/layers.py | 7 +- 2 files changed, 78 insertions(+), 144 deletions(-) diff --git a/doc/api/v2/config/layer.rst b/doc/api/v2/config/layer.rst index 05817ec854..2a02baf17b 100644 --- a/doc/api/v2/config/layer.rst +++ b/doc/api/v2/config/layer.rst @@ -11,8 +11,7 @@ Data layer data ---- -.. automodule:: paddle.v2.layer - :members: data +.. autoclass:: paddle.v2.layer.data :noindex: Fully Connected Layers @@ -22,14 +21,12 @@ Fully Connected Layers fc -- -.. automodule:: paddle.v2.layer - :members: fc +.. autoclass:: paddle.v2.layer.fc :noindex: selective_fc ------------ -.. automodule:: paddle.v2.layer - :members: selective_fc +.. autoclass:: paddle.v2.layer.selective_fc :noindex: Conv Layers @@ -37,34 +34,29 @@ Conv Layers conv_operator ------------- -.. automodule:: paddle.v2.layer - :members: conv_operator +.. autoclass:: paddle.v2.layer.conv_operator :noindex: conv_projection --------------- -.. automodule:: paddle.v2.layer - :members: conv_projection +.. autoclass:: paddle.v2.layer.conv_projection :noindex: conv_shift ---------- -.. automodule:: paddle.v2.layer - :members: conv_shift +.. autoclass:: paddle.v2.layer.conv_shift :noindex: img_conv -------- -.. automodule:: paddle.v2.layer - :members: img_conv +.. autoclass:: paddle.v2.layer.img_conv :noindex: .. _api_v2.layer_context_projection: context_projection ------------------ -.. automodule:: paddle.v2.layer - :members: context_projection +.. autoclass:: paddle.v2.layer.context_projection :noindex: Image Pooling Layer @@ -72,20 +64,17 @@ Image Pooling Layer img_pool -------- -.. automodule:: paddle.v2.layer - :members: img_pool +.. autoclass:: paddle.v2.layer.img_pool :noindex: spp --- -.. automodule:: paddle.v2.layer - :members: spp +.. autoclass:: paddle.v2.layer.spp :noindex: maxout ------ -.. automodule:: paddle.v2.layer - :members: maxout +.. autoclass:: paddle.v2.layer.maxout :noindex: Norm Layer @@ -93,26 +82,22 @@ Norm Layer img_cmrnorm ----------- -.. automodule:: paddle.v2.layer - :members: img_cmrnorm +.. autoclass:: paddle.v2.layer.img_cmrnorm :noindex: batch_norm ---------- -.. automodule:: paddle.v2.layer - :members: batch_norm +.. autoclass:: paddle.v2.layer.batch_norm :noindex: sum_to_one_norm --------------- -.. automodule:: paddle.v2.layer - :members: sum_to_one_norm +.. autoclass:: paddle.v2.layer.sum_to_one_norm :noindex: cross_channel_norm ------------------ -.. automodule:: paddle.v2.layer - :members: cross_channel_norm +.. autoclass:: paddle.v2.layer.cross_channel_norm :noindex: Recurrent Layers @@ -120,20 +105,17 @@ Recurrent Layers recurrent --------- -.. automodule:: paddle.v2.layer - :members: recurrent +.. autoclass:: paddle.v2.layer.recurrent :noindex: lstmemory --------- -.. automodule:: paddle.v2.layer - :members: lstmemory +.. autoclass:: paddle.v2.layer.lstmemory :noindex: grumemory --------- -.. automodule:: paddle.v2.layer - :members: grumemory +.. autoclass:: paddle.v2.layer.grumemory :noindex: Recurrent Layer Group @@ -141,38 +123,32 @@ Recurrent Layer Group memory ------ -.. automodule:: paddle.v2.layer - :members: memory +.. autoclass:: paddle.v2.layer.memory :noindex: recurrent_group --------------- -.. automodule:: paddle.v2.layer - :members: recurrent_group +.. autoclass:: paddle.v2.layer.recurrent_group :noindex: lstm_step --------- -.. automodule:: paddle.v2.layer - :members: lstm_step +.. autoclass:: paddle.v2.layer.lstm_step :noindex: gru_step -------- -.. automodule:: paddle.v2.layer - :members: gru_step +.. autoclass:: paddle.v2.layer.gru_step :noindex: beam_search ------------ -.. automodule:: paddle.v2.layer - :members: beam_search +.. autoclass:: paddle.v2.layer.beam_search :noindex: get_output ---------- -.. automodule:: paddle.v2.layer - :members: get_output +.. autoclass:: paddle.v2.layer.get_output :noindex: Mixed Layer @@ -182,59 +158,50 @@ Mixed Layer mixed ----- -.. automodule:: paddle.v2.layer - :members: mixed +.. autoclass:: paddle.v2.layer.mixed :noindex: .. _api_v2.layer_embedding: embedding --------- -.. automodule:: paddle.v2.layer - :members: embedding +.. autoclass:: paddle.v2.layer.embedding :noindex: scaling_projection ------------------ -.. automodule:: paddle.v2.layer - :members: scaling_projection +.. autoclass:: paddle.v2.layer.scaling_projection :noindex: dotmul_projection ----------------- -.. automodule:: paddle.v2.layer - :members: dotmul_projection +.. autoclass:: paddle.v2.layer.dotmul_projection :noindex: dotmul_operator --------------- -.. automodule:: paddle.v2.layer - :members: dotmul_operator +.. autoclass:: paddle.v2.layer.dotmul_operator :noindex: full_matrix_projection ---------------------- -.. automodule:: paddle.v2.layer - :members: full_matrix_projection +.. autoclass:: paddle.v2.layer.full_matrix_projection :noindex: identity_projection ------------------- -.. automodule:: paddle.v2.layer - :members: identity_projection +.. autoclass:: paddle.v2.layer.identity_projection :noindex: table_projection ---------------- -.. automodule:: paddle.v2.layer - :members: table_projection +.. autoclass:: paddle.v2.layer.table_projection :noindex: trans_full_matrix_projection ---------------------------- -.. automodule:: paddle.v2.layer - :members: trans_full_matrix_projection +.. autoclass:: paddle.v2.layer.trans_full_matrix_projection :noindex: Aggregate Layers @@ -244,36 +211,31 @@ Aggregate Layers pooling ------- -.. automodule:: paddle.v2.layer - :members: pooling +.. autoclass:: paddle.v2.layer.pooling :noindex: .. _api_v2.layer_last_seq: last_seq -------- -.. automodule:: paddle.v2.layer - :members: last_seq +.. autoclass:: paddle.v2.layer.last_seq :noindex: .. _api_v2.layer_first_seq: first_seq --------- -.. automodule:: paddle.v2.layer - :members: first_seq +.. autoclass:: paddle.v2.layer.first_seq :noindex: concat ------ -.. automodule:: paddle.v2.layer - :members: concat +.. autoclass:: paddle.v2.layer.concat :noindex: seq_concat ---------- -.. automodule:: paddle.v2.layer - :members: seq_concat +.. autoclass:: paddle.v2.layer.seq_concat :noindex: Reshaping Layers @@ -281,34 +243,29 @@ Reshaping Layers block_expand ------------ -.. automodule:: paddle.v2.layer - :members: block_expand +.. autoclass:: paddle.v2.layer.block_expand :noindex: .. _api_v2.layer_expand: expand ------ -.. automodule:: paddle.v2.layer - :members: expand +.. autoclass:: paddle.v2.layer.expand :noindex: repeat ------ -.. automodule:: paddle.v2.layer - :members: repeat +.. autoclass:: paddle.v2.layer.repeat :noindex: rotate ------ -.. automodule:: paddle.v2.layer - :members: rotate +.. autoclass:: paddle.v2.layer.rotate :noindex: seq_reshape ----------- -.. automodule:: paddle.v2.layer - :members: seq_reshape +.. autoclass:: paddle.v2.layer.seq_reshape :noindex: Math Layers @@ -316,64 +273,54 @@ Math Layers addto ----- -.. automodule:: paddle.v2.layer - :members: addto +.. autoclass:: paddle.v2.layer.addto :noindex: linear_comb ----------- -.. automodule:: paddle.v2.layer - :members: linear_comb +.. autoclass:: paddle.v2.layer.linear_comb :noindex: interpolation ------------- -.. automodule:: paddle.v2.layer - :members: interpolation +.. autoclass:: paddle.v2.layer.interpolation :noindex: bilinear_interp --------------- -.. automodule:: paddle.v2.layer - :members: bilinear_interp +.. autoclass:: paddle.v2.layer.bilinear_interp :noindex: power ----- -.. automodule:: paddle.v2.layer - :members: power +.. autoclass:: paddle.v2.layer.power :noindex: scaling ------- -.. automodule:: paddle.v2.layer - :members: scaling +.. autoclass:: paddle.v2.layer.scaling :noindex: slope_intercept --------------- -.. automodule:: paddle.v2.layer - :members: slope_intercept +.. autoclass:: paddle.v2.layer.slope_intercept :noindex: tensor ------ -.. automodule:: paddle.v2.layer - :members: tensor +.. autoclass:: paddle.v2.layer.tensor :noindex: .. _api_v2.layer_cos_sim: cos_sim ------- -.. automodule:: paddle.v2.layer - :members: cos_sim +.. autoclass:: paddle.v2.layer.cos_sim :noindex: trans ----- -.. automodule:: paddle.v2.layer - :members: trans +.. autoclass:: paddle.v2.layer.trans :noindex: Sampling Layers @@ -381,14 +328,12 @@ Sampling Layers maxid ----- -.. automodule:: paddle.v2.layer - :members: maxid +.. autoclass:: paddle.v2.layer.max_id :noindex: sampling_id ----------- -.. automodule:: paddle.v2.layer - :members: sampling_id +.. autoclass:: paddle.v2.layer.sampling_id :noindex: Slicing and Joining Layers @@ -396,8 +341,7 @@ Slicing and Joining Layers pad ---- -.. automodule:: paddle.v2.layer - :members: pad +.. autoclass:: paddle.v2.layer.pad :noindex: .. _api_v2.layer_costs: @@ -407,80 +351,72 @@ Cost Layers cross_entropy_cost ------------------ -.. automodule:: paddle.v2.layer - :members: cross_entropy_cost +.. autoclass:: paddle.v2.layer.cross_entropy_cost :noindex: cross_entropy_with_selfnorm_cost -------------------------------- -.. automodule:: paddle.v2.layer - :members: cross_entropy_with_selfnorm_cost +.. autoclass:: paddle.v2.layer.cross_entropy_with_selfnorm_cost :noindex: multi_binary_label_cross_entropy_cost ------------------------------------- -.. automodule:: paddle.v2.layer - :members: multi_binary_label_cross_entropy_cost +.. autoclass:: paddle.v2.layer.multi_binary_label_cross_entropy_cost :noindex: huber_cost ---------- -.. automodule:: paddle.v2.layer - :members: huber_cost +.. autoclass:: paddle.v2.layer.huber_cost :noindex: lambda_cost ----------- -.. automodule:: paddle.v2.layer - :members: lambda_cost +.. autoclass:: paddle.v2.layer.lambda_cost + :noindex: + +mse_cost +-------- +.. autoclass:: paddle.v2.layer.mse_cost :noindex: rank_cost --------- -.. automodule:: paddle.v2.layer - :members: rank_cost +.. autoclass:: paddle.v2.layer.rank_cost :noindex: sum_cost --------- -.. automodule:: paddle.v2.layer - :members: sum_cost +.. autoclass:: paddle.v2.layer.sum_cost :noindex: crf --- -.. automodule:: paddle.v2.layer - :members: crf +.. autoclass:: paddle.v2.layer.crf :noindex: crf_decoding ------------ -.. automodule:: paddle.v2.layer - :members: crf_decoding +.. autoclass:: paddle.v2.layer.crf_decoding :noindex: ctc --- -.. automodule:: paddle.v2.layer - :members: ctc +.. autoclass:: paddle.v2.layer.ctc :noindex: warp_ctc -------- -.. automodule:: paddle.v2.layer - :members: warp_ctc +.. autoclass:: paddle.v2.layer.warp_ctc :noindex: nce --- -.. automodule:: paddle.v2.layer - :members: nce +.. autoclass:: paddle.v2.layer.nce :noindex: hsigmoid --------- -.. automodule:: paddle.v2.layer - :members: hsigmoid +.. autoclass:: paddle.v2.layer.hsigmoid :noindex: Check Layer @@ -488,6 +424,5 @@ Check Layer eos --- -.. automodule:: paddle.v2.layer - :members: eos +.. autoclass:: paddle.v2.layer.eos :noindex: diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index 97db3c2d4c..f906126d87 100755 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -567,7 +567,7 @@ def dotmul_operator(a=None, b=None, scale=1, **kwargs): DotMulOperator takes two inputs and performs element-wise multiplication: .. math:: - out.row[i] += scale * (x.row[i] .* y.row[i]) + out.row[i] += scale * (a.row[i] .* b.row[i]) where :math:`.*` means element-wise multiplication, and scale is a config scalar, its default value is one. @@ -576,7 +576,7 @@ def dotmul_operator(a=None, b=None, scale=1, **kwargs): .. code-block:: python - op = dotmul_operator(x=layer1, y=layer2, scale=0.5) + op = dotmul_operator(a=layer1, b=layer2, scale=0.5) :param a: Input layer1 :type a: LayerOutput @@ -3689,8 +3689,7 @@ def mse_cost(input, label, weight=None, name=None, layer_attr=None): .. math:: - $\frac{1}{N}\sum_{i=1}^N(t _i- y_i)^2$ - + \frac{1}{N}\sum_{i=1}^N(t_i-y_i)^2 :param name: layer name. :type name: basestring From feccfc8333309e64623ec522d48284182c493117 Mon Sep 17 00:00:00 2001 From: qijun Date: Thu, 20 Apr 2017 14:27:02 +0800 Subject: [PATCH 06/19] fix with_doc build option --- paddle/scripts/docker/build.sh | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh index 16f669bd77..5156cf9147 100644 --- a/paddle/scripts/docker/build.sh +++ b/paddle/scripts/docker/build.sh @@ -29,7 +29,7 @@ rm *.deb 2>/dev/null || true cmake .. \ -DCMAKE_BUILD_TYPE=Release \ - -DWITH_DOC=${WITH_DOC:-OFF} \ + -DWITH_DOC=OFF \ -DWITH_GPU=${WITH_GPU:-OFF} \ -DWITH_AVX=${WITH_AVX:-OFF} \ -DWITH_SWIG_PY=ON \ @@ -100,6 +100,18 @@ RUN dpkg -i /usr/local/opt/paddle/deb/*.deb && \ rm -f /usr/local/opt/paddle/deb/*.deb && \ pip install /usr/opt/paddle/share/wheels/*.whl && \ paddle version +if [ ${WITH_DOC} == "ON" ]; then + mkdir -p /paddle/build_doc + pushd /paddle/build_doc + cmake .. \ + -DWITH_DOC=ON \ + -DWITH_GPU=OFF \ + -DWITH_AVX=${WITH_AVX:-OFF} \ + -DWITH_SWIG_PY=ON \ + -DWITH_STYLE_CHECK=OFF + make paddle_docs + popd +fi ${CPU_DOCKER_PYTHON_HOME_ENV} ${DOCKERFILE_CUDNN_DSO} ${DOCKERFILE_GPU_ENV} From 99ae4dae46a8cb4b2276f35619382ae51be06e3a Mon Sep 17 00:00:00 2001 From: qijun Date: Thu, 20 Apr 2017 14:30:40 +0800 Subject: [PATCH 07/19] add build paddle_docs_cn --- paddle/scripts/docker/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh index 5156cf9147..50b083fb39 100644 --- a/paddle/scripts/docker/build.sh +++ b/paddle/scripts/docker/build.sh @@ -109,7 +109,7 @@ if [ ${WITH_DOC} == "ON" ]; then -DWITH_AVX=${WITH_AVX:-OFF} \ -DWITH_SWIG_PY=ON \ -DWITH_STYLE_CHECK=OFF - make paddle_docs + make paddle_docs paddle_docs_cn popd fi ${CPU_DOCKER_PYTHON_HOME_ENV} From 4aabe597f62f19bd0c4478884d56a3db93740c36 Mon Sep 17 00:00:00 2001 From: qijun Date: Thu, 20 Apr 2017 14:38:10 +0800 Subject: [PATCH 08/19] fix dockerfile typo error --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 97947adf45..49dcc2ad96 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ ARG WITH_DOC ARG WITH_STYLE_CHECK ENV WOBOQ OFF -ENV WITH_GPU=${WITH_AVX:-OFF} +ENV WITH_GPU=${WITH_GPU:-OFF} ENV WITH_AVX=${WITH_AVX:-ON} ENV WITH_DOC=${WITH_DOC:-OFF} ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF} From 8dbbcc1fc11eb4403cabebe7577fa71f3508bd76 Mon Sep 17 00:00:00 2001 From: qijun Date: Thu, 20 Apr 2017 14:47:21 +0800 Subject: [PATCH 09/19] add install doxygen --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 49dcc2ad96..84f78ac5dd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,7 +27,7 @@ RUN apt-get update && \ apt-get install -y wget unzip tar xz-utils bzip2 gzip coreutils && \ apt-get install -y curl sed grep graphviz libjpeg-dev zlib1g-dev && \ apt-get install -y python-numpy python-matplotlib gcc g++ gfortran && \ - apt-get install -y automake locales clang-format-3.8 swig && \ + apt-get install -y automake locales clang-format-3.8 swig doxygen&& \ apt-get clean -y # git credential to skip password typing From 99567d4eb1eb8144cdbf5954eb03e32414867566 Mon Sep 17 00:00:00 2001 From: qijun Date: Thu, 20 Apr 2017 14:52:09 +0800 Subject: [PATCH 10/19] format code --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 84f78ac5dd..eaf472221a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,7 +27,7 @@ RUN apt-get update && \ apt-get install -y wget unzip tar xz-utils bzip2 gzip coreutils && \ apt-get install -y curl sed grep graphviz libjpeg-dev zlib1g-dev && \ apt-get install -y python-numpy python-matplotlib gcc g++ gfortran && \ - apt-get install -y automake locales clang-format-3.8 swig doxygen&& \ + apt-get install -y automake locales clang-format-3.8 swig doxygen && \ apt-get clean -y # git credential to skip password typing From f5d0970e13caeb097bdeb78378ab9ff733718208 Mon Sep 17 00:00:00 2001 From: qijun Date: Thu, 20 Apr 2017 17:08:36 +0800 Subject: [PATCH 11/19] fix ipython version --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index eaf472221a..f12be36ceb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -43,7 +43,8 @@ RUN pip install --upgrade pip && \ pip install -U wheel pillow BeautifulSoup && \ pip install -U docopt PyYAML sphinx && \ pip install -U sphinx-rtd-theme==0.1.9 recommonmark && \ - pip install pre-commit 'requests==2.9.2' 'ipykernel==4.6.0' 'jupyter==1.0.0' + pip install pre-commit 'requests==2.9.2' 'ipython==5.3.0' && \ + pip install 'ipykernel==4.6.0' 'jupyter==1.0.0' RUN curl -sSL https://cmake.org/files/v3.4/cmake-3.4.1.tar.gz | tar -xz && \ cd cmake-3.4.1 && ./bootstrap && make -j `nproc` && make install && \ From 0344c086e53a92822654741773ec839362c2a4f5 Mon Sep 17 00:00:00 2001 From: qijun Date: Thu, 20 Apr 2017 19:19:44 +0800 Subject: [PATCH 12/19] fix build.sh --- paddle/scripts/docker/build.sh | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh index 50b083fb39..c1e3f39888 100644 --- a/paddle/scripts/docker/build.sh +++ b/paddle/scripts/docker/build.sh @@ -42,7 +42,20 @@ if [[ ${RUN_TEST:-OFF} == "ON" ]]; then make coveralls fi make install +pip install /usr/opt/paddle/share/wheels/*.whl +if [ ${WITH_DOC} == "ON" ]; then + mkdir -p /paddle/build_doc + pushd /paddle/build_doc + cmake .. \ + -DWITH_DOC=ON \ + -DWITH_GPU=OFF \ + -DWITH_AVX=${WITH_AVX:-OFF} \ + -DWITH_SWIG_PY=ON \ + -DWITH_STYLE_CHECK=OFF + make paddle_docs paddle_docs_cn + popd +fi # generate deb package for current build # FIXME(typhoonzero): should we remove paddle/scripts/deb ? # FIXME: CPACK_DEBIAN_PACKAGE_DEPENDS removes all dev dependencies, must @@ -100,18 +113,6 @@ RUN dpkg -i /usr/local/opt/paddle/deb/*.deb && \ rm -f /usr/local/opt/paddle/deb/*.deb && \ pip install /usr/opt/paddle/share/wheels/*.whl && \ paddle version -if [ ${WITH_DOC} == "ON" ]; then - mkdir -p /paddle/build_doc - pushd /paddle/build_doc - cmake .. \ - -DWITH_DOC=ON \ - -DWITH_GPU=OFF \ - -DWITH_AVX=${WITH_AVX:-OFF} \ - -DWITH_SWIG_PY=ON \ - -DWITH_STYLE_CHECK=OFF - make paddle_docs paddle_docs_cn - popd -fi ${CPU_DOCKER_PYTHON_HOME_ENV} ${DOCKERFILE_CUDNN_DSO} ${DOCKERFILE_GPU_ENV} From 2925f0089a92f345a17725abd28e2cde02252f2e Mon Sep 17 00:00:00 2001 From: qijun Date: Thu, 20 Apr 2017 19:36:15 +0800 Subject: [PATCH 13/19] fix wrong path of pip install *whl --- paddle/scripts/docker/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh index c1e3f39888..6f7b97021d 100644 --- a/paddle/scripts/docker/build.sh +++ b/paddle/scripts/docker/build.sh @@ -42,7 +42,7 @@ if [[ ${RUN_TEST:-OFF} == "ON" ]]; then make coveralls fi make install -pip install /usr/opt/paddle/share/wheels/*.whl +pip install /usr/local/opt/paddle/share/wheels/*.whl if [ ${WITH_DOC} == "ON" ]; then mkdir -p /paddle/build_doc @@ -111,7 +111,7 @@ ADD build/*.deb /usr/local/opt/paddle/deb/ # run paddle version to install python packages first RUN dpkg -i /usr/local/opt/paddle/deb/*.deb && \ rm -f /usr/local/opt/paddle/deb/*.deb && \ - pip install /usr/opt/paddle/share/wheels/*.whl && \ + pip install /usr/local/opt/paddle/share/wheels/*.whl && \ paddle version ${CPU_DOCKER_PYTHON_HOME_ENV} ${DOCKERFILE_CUDNN_DSO} From 07d2e0f96fe569e73fd31ab9451cd3a1c4da9d39 Mon Sep 17 00:00:00 2001 From: qijun Date: Thu, 20 Apr 2017 19:48:54 +0800 Subject: [PATCH 14/19] add build_doc dir to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index ee7c6ec370..2b30f7938c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ *.DS_Store build/ +build_doc/ *.user .vscode From 81e0f5a4e25a20dcc198df26d8bc48e8df4c3f36 Mon Sep 17 00:00:00 2001 From: qijun Date: Fri, 21 Apr 2017 14:29:19 +0800 Subject: [PATCH 15/19] correct with_testing option --- paddle/scripts/docker/build.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh index 6f7b97021d..83cc30d3c2 100644 --- a/paddle/scripts/docker/build.sh +++ b/paddle/scripts/docker/build.sh @@ -35,11 +35,11 @@ cmake .. \ -DWITH_SWIG_PY=ON \ -DCUDNN_ROOT=/usr/ \ -DWITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF} \ - -DON_COVERALLS=${WITH_TEST:-OFF} \ + -DWITH_TESTING=${WITH_TESTING:-OFF} \ -DCMAKE_EXPORT_COMPILE_COMMANDS=ON make -j `nproc` -if [[ ${RUN_TEST:-OFF} == "ON" ]]; then - make coveralls +if [ ${WITH_TESTING:-OFF} == "ON" ] && [ ${RUN_TEST:-OFF} == "ON" ] ; then + make test fi make install pip install /usr/local/opt/paddle/share/wheels/*.whl From 190e429dfdac0c206b86bdd7bc515c06dfdb6390 Mon Sep 17 00:00:00 2001 From: qijun Date: Fri, 21 Apr 2017 14:35:05 +0800 Subject: [PATCH 16/19] add some comments --- paddle/scripts/docker/README.md | 2 +- paddle/scripts/docker/build.sh | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/paddle/scripts/docker/README.md b/paddle/scripts/docker/README.md index 132f8cd8aa..b45d92507c 100644 --- a/paddle/scripts/docker/README.md +++ b/paddle/scripts/docker/README.md @@ -94,7 +94,7 @@ docker build -t paddle:dev --build-arg UBUNTU_MIRROR=mirror://mirrors.ubuntu.com Given the development image `paddle:dev`, the following command builds PaddlePaddle from the source tree on the development computer (host): ```bash -docker run --rm -v $PWD:/paddle -e "WITH_GPU=OFF" -e "WITH_AVX=ON" -e "WITH_TEST=OFF" -e "RUN_TEST=OFF" paddle:dev +docker run --rm -v $PWD:/paddle -e "WITH_GPU=OFF" -e "WITH_AVX=ON" -e "WITH_TESTING=OFF" -e "RUN_TEST=OFF" paddle:dev ``` This command mounts the source directory on the host into `/paddle` in the container, so the default entry point of `paddle:dev`, `build.sh`, could build the source code with possible local changes. When it writes to `/paddle/build` in the container, it writes to `$PWD/build` on the host indeed. diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh index 83cc30d3c2..1613d970bc 100644 --- a/paddle/scripts/docker/build.sh +++ b/paddle/scripts/docker/build.sh @@ -44,6 +44,8 @@ fi make install pip install /usr/local/opt/paddle/share/wheels/*.whl +# Since python v2 api import py_paddle module, the generation of paddle docs +# depend on paddle's compilation and installation if [ ${WITH_DOC} == "ON" ]; then mkdir -p /paddle/build_doc pushd /paddle/build_doc From 2c4892c968560dcceec72f8cbc92d8ddc841aa2a Mon Sep 17 00:00:00 2001 From: qijun Date: Fri, 21 Apr 2017 18:56:11 +0800 Subject: [PATCH 17/19] find whl path --- paddle/scripts/docker/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh index 1613d970bc..185aa05077 100644 --- a/paddle/scripts/docker/build.sh +++ b/paddle/scripts/docker/build.sh @@ -113,7 +113,7 @@ ADD build/*.deb /usr/local/opt/paddle/deb/ # run paddle version to install python packages first RUN dpkg -i /usr/local/opt/paddle/deb/*.deb && \ rm -f /usr/local/opt/paddle/deb/*.deb && \ - pip install /usr/local/opt/paddle/share/wheels/*.whl && \ + find /usr/ -name '*paddle-*.whl' | xargs pip install && \ paddle version ${CPU_DOCKER_PYTHON_HOME_ENV} ${DOCKERFILE_CUDNN_DSO} From 7b16103f87332f07be2282937194d737fa397c14 Mon Sep 17 00:00:00 2001 From: qijun Date: Mon, 24 Apr 2017 10:33:56 +0800 Subject: [PATCH 18/19] update comments --- paddle/scripts/docker/build.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh index 185aa05077..73f885c1e7 100644 --- a/paddle/scripts/docker/build.sh +++ b/paddle/scripts/docker/build.sh @@ -44,8 +44,9 @@ fi make install pip install /usr/local/opt/paddle/share/wheels/*.whl -# Since python v2 api import py_paddle module, the generation of paddle docs -# depend on paddle's compilation and installation +# To build documentation, we need to run cmake twice. +# This awkwardness is due to https://github.com/PaddlePaddle/Paddle/issues/1854. +# It also describes a solution. if [ ${WITH_DOC} == "ON" ]; then mkdir -p /paddle/build_doc pushd /paddle/build_doc From 7e2d6e6198f92f1c532daaaf93115617481db634 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Mon, 24 Apr 2017 12:40:04 +0800 Subject: [PATCH 19/19] Start release note --- RELEASE.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/RELEASE.md b/RELEASE.md index a8a245ab44..f1588cb15b 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,3 +1,12 @@ +# Release v0.10.0 + +## New Features + +## Improvements + +## Bug Fixes + + # Release v0.9.0 ## New Features: