From aca08255d2c2a9c7ca06ce4c53765b366f66a227 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Fri, 25 Nov 2016 22:05:45 -0800 Subject: [PATCH 1/7] Allow docker build to build from local Git commit. --- paddle/scripts/docker/Dockerfile.m4 | 40 +++++++++++++++---- paddle/scripts/docker/build.sh | 40 +++++-------------- paddle/scripts/docker/generate.sh | 60 ++++++----------------------- 3 files changed, 53 insertions(+), 87 deletions(-) mode change 100644 => 100755 paddle/scripts/docker/build.sh mode change 100644 => 100755 paddle/scripts/docker/generate.sh diff --git a/paddle/scripts/docker/Dockerfile.m4 b/paddle/scripts/docker/Dockerfile.m4 index e14493ed9e..7f08e66653 100644 --- a/paddle/scripts/docker/Dockerfile.m4 +++ b/paddle/scripts/docker/Dockerfile.m4 @@ -1,12 +1,36 @@ FROM PADDLE_BASE_IMAGE MAINTAINER PaddlePaddle Dev Team -COPY build.sh /root/ -ENV GIT_CHECKOUT=v0.9.0a0 + +# It is good to run apt-get install with Dockerfile RUN directive, +# because if the following invocation to /root/build.sh fails, `docker +# build` wouldn't have to re-install packages after we fix +# /root/build.sh. +RUN apt-get update && \ + apt-get install -y cmake libprotobuf-dev protobuf-compiler git \ + libgoogle-glog-dev libgflags-dev libatlas-dev libatlas3-base g++ m4 python-pip \ + python-protobuf python-numpy python-dev swig openssh-server \ + wget unzip python-matplotlib tar xz-utils bzip2 gzip coreutils \ + sed grep graphviz libjpeg-dev zlib1g-dev doxygen && \ + apt-get clean -y +RUN pip install BeautifulSoup docopt PyYAML pillow \ + 'sphinx>=1.4.0' sphinx_rtd_theme breathe recommonmark + ENV WITH_GPU=PADDLE_WITH_GPU -ENV IS_DEVEL=PADDLE_IS_DEVEL -ENV WITH_DEMO=PADDLE_WITH_DEMO -ENV PIP_INSTALL_ARGS "" -ENV PIP_GENERAL_ARGS "" -ENV USE_UBUNTU_MIRROR OFF ENV WITH_AVX=PADDLE_WITH_AVX -RUN cd /root/ && bash build.sh + +RUN mkdir /paddle +COPY . /paddle/ +COPY paddle/scripts/docker/build.sh /root/ +RUN /root/build.sh + +RUN echo 'export LD_LIBRARY_PATH=/usr/lib64:${LD_LIBRARY_PATH}' >> /etc/profile +RUN pip install /usr/local/opt/paddle/share/wheels/*.whl +RUN paddle version # print version after build + +# Configure OpenSSH server. c.f. https://docs.docker.com/engine/examples/running_ssh_service +RUN mkdir /var/run/sshd +RUN echo 'root:root' | chpasswd +RUN sed -ri 's/^PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config +RUN sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config +EXPOSE 22 +CMD ["/usr/sbin/sshd", "-D"] diff --git a/paddle/scripts/docker/build.sh b/paddle/scripts/docker/build.sh old mode 100644 new mode 100755 index ec5f3bd967..8e2e26b6ba --- a/paddle/scripts/docker/build.sh +++ b/paddle/scripts/docker/build.sh @@ -7,43 +7,21 @@ function abort(){ trap 'abort' 0 set -e -if [ ${USE_UBUNTU_MIRROR} == "ON" ]; then - sed -i 's#http://archive\.ubuntu\.com/ubuntu/#mirror://mirrors\.ubuntu\.com/mirrors\.txt#g'\ - /etc/apt/sources.list -fi -apt-get update -apt-get install -y cmake libprotobuf-dev protobuf-compiler git \ - libgoogle-glog-dev libgflags-dev libatlas-dev libatlas3-base g++ m4 python-pip\ - python-protobuf python-numpy python-dev swig if [ ${WITH_GPU} == 'ON' ]; then ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so /usr/lib/libcudnn.so fi -cd ~ -git clone https://github.com/baidu/Paddle.git paddle -cd paddle -git checkout ${GIT_CHECKOUT} -mkdir build -cd build -cmake .. -DWITH_DOC=OFF -DWITH_GPU=${WITH_GPU} -DWITH_SWIG_PY=ON\ - -DCUDNN_ROOT=/usr/ -DWITH_STYLE_CHECK=OFF -DWITH_AVX=${WITH_AVX} +mkdir -p /paddle/build # -p means no error if exists +cd /paddle/build +cmake .. \ + -DWITH_DOC=ON \ + -DWITH_GPU=${WITH_GPU} \ + -DWITH_AVX=${WITH_AVX} \ + -DWITH_SWIG_PY=ON \ + -DCUDNN_ROOT=/usr/ \ + -DWITH_STYLE_CHECK=OFF make -j `nproc` -# because durning make install, there are several warning, so set +e, do not cause abort make install -echo 'export LD_LIBRARY_PATH=/usr/lib64:${LD_LIBRARY_PATH}' >> /etc/profile -pip ${PIP_GENERAL_ARGS} install ${PIP_INSTALL_ARGS} /usr/local/opt/paddle/share/wheels/*.whl -paddle version # print version after build -if [ ${WITH_DEMO} == "ON" ]; then - apt-get install -y wget unzip perl python-matplotlib tar xz-utils bzip2 gzip coreutils\ - sed grep graphviz libjpeg-dev zlib1g-dev - pip ${PIP_GENERAL_ARGS} install ${PIP_INSTALL_ARGS} BeautifulSoup docopt \ - PyYAML pillow -fi -if [ ${IS_DEVEL} == "OFF" ]; then # clean build packages. - cd ~ - rm -rf paddle -fi -apt-get clean -y trap : 0 diff --git a/paddle/scripts/docker/generate.sh b/paddle/scripts/docker/generate.sh old mode 100644 new mode 100755 index 2ad7527db1..b808a62ec2 --- a/paddle/scripts/docker/generate.sh +++ b/paddle/scripts/docker/generate.sh @@ -1,60 +1,24 @@ #!/bin/bash + set -e cd `dirname $0` -m4 -DPADDLE_WITH_GPU=OFF -DPADDLE_IS_DEVEL=OFF -DPADDLE_WITH_DEMO=OFF \ - -DPADDLE_BASE_IMAGE=ubuntu:14.04 -DPADDLE_WITH_AVX=ON\ + +m4 -DPADDLE_WITH_GPU=OFF \ + -DPADDLE_WITH_AVX=ON \ + -DPADDLE_BASE_IMAGE=ubuntu:14.04 \ Dockerfile.m4 > Dockerfile.cpu -m4 -DPADDLE_WITH_GPU=OFF -DPADDLE_IS_DEVEL=OFF -DPADDLE_WITH_DEMO=OFF \ - -DPADDLE_BASE_IMAGE=ubuntu:14.04 -DPADDLE_WITH_AVX=OFF\ +m4 -DPADDLE_WITH_GPU=OFF \ + -DPADDLE_WITH_AVX=OFF \ + -DPADDLE_BASE_IMAGE=ubuntu:14.04 \ Dockerfile.m4 > Dockerfile.cpu-noavx -m4 -DPADDLE_WITH_GPU=OFF -DPADDLE_IS_DEVEL=ON -DPADDLE_WITH_DEMO=OFF \ - -DPADDLE_BASE_IMAGE=ubuntu:14.04 -DPADDLE_WITH_AVX=OFF\ - Dockerfile.m4 > Dockerfile.cpu-noavx-devel - -m4 -DPADDLE_WITH_GPU=OFF -DPADDLE_IS_DEVEL=ON -DPADDLE_WITH_DEMO=OFF \ - -DPADDLE_BASE_IMAGE=ubuntu:14.04 -DPADDLE_WITH_AVX=ON\ - Dockerfile.m4 > Dockerfile.cpu-devel - - -m4 -DPADDLE_WITH_GPU=OFF -DPADDLE_IS_DEVEL=ON -DPADDLE_WITH_DEMO=ON \ - -DPADDLE_BASE_IMAGE=ubuntu:14.04 -DPADDLE_WITH_AVX=ON\ - Dockerfile.m4 > Dockerfile.cpu-demo - -m4 -DPADDLE_WITH_GPU=OFF -DPADDLE_IS_DEVEL=ON -DPADDLE_WITH_DEMO=ON \ - -DPADDLE_BASE_IMAGE=ubuntu:14.04 -DPADDLE_WITH_AVX=OFF\ - Dockerfile.m4 > Dockerfile.cpu-noavx-demo - - -m4 -DPADDLE_WITH_GPU=ON -DPADDLE_IS_DEVEL=OFF -DPADDLE_WITH_DEMO=OFF \ - -DPADDLE_BASE_IMAGE=nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 \ +m4 -DPADDLE_WITH_GPU=ON \ -DPADDLE_WITH_AVX=ON \ - Dockerfile.m4 > Dockerfile.gpu - -m4 -DPADDLE_WITH_GPU=ON -DPADDLE_IS_DEVEL=OFF -DPADDLE_WITH_DEMO=OFF \ -DPADDLE_BASE_IMAGE=nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 \ - -DPADDLE_WITH_AVX=OFF \ - Dockerfile.m4 > Dockerfile.gpu-noavx - - -m4 -DPADDLE_WITH_GPU=ON -DPADDLE_IS_DEVEL=ON -DPADDLE_WITH_DEMO=OFF \ - -DPADDLE_BASE_IMAGE=nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 \ - -DPADDLE_WITH_AVX=ON \ - Dockerfile.m4 > Dockerfile.gpu-devel + Dockerfile.m4 > Dockerfile.gpu -m4 -DPADDLE_WITH_GPU=ON -DPADDLE_IS_DEVEL=ON -DPADDLE_WITH_DEMO=OFF \ - -DPADDLE_BASE_IMAGE=nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 \ +m4 -DPADDLE_WITH_GPU=ON \ -DPADDLE_WITH_AVX=OFF \ - Dockerfile.m4 > Dockerfile.gpu-noavx-devel - -m4 -DPADDLE_WITH_GPU=ON -DPADDLE_IS_DEVEL=ON -DPADDLE_WITH_DEMO=ON \ -DPADDLE_BASE_IMAGE=nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 \ - -DPADDLE_WITH_AVX=ON \ - Dockerfile.m4 > Dockerfile.gpu-demo - - -m4 -DPADDLE_WITH_GPU=ON -DPADDLE_IS_DEVEL=ON -DPADDLE_WITH_DEMO=ON \ - -DPADDLE_BASE_IMAGE=nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 \ - -DPADDLE_WITH_AVX=OFF \ - Dockerfile.m4 > Dockerfile.gpu-noavx-demo + Dockerfile.m4 > Dockerfile.gpu-noavx From 4e2071f47e488f9f7fd74fbe4af077ffd0edd9da Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Fri, 25 Nov 2016 22:06:16 -0800 Subject: [PATCH 2/7] Make cmake derive Paddle version from Git tag or Git commit ID --- CMakeLists.txt | 50 +++++++++++++++++++++++--------------------------- 1 file changed, 23 insertions(+), 27 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index af193c27ae..3ad8663578 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,10 +1,6 @@ cmake_minimum_required(VERSION 2.8) project(paddle CXX C) -set(PADDLE_MAJOR_VERSION 0) -set(PADDLE_MINOR_VERSION 9) -set(PADDLE_PATCH_VERSION 0a0) -set(PADDLE_VERSION ${PADDLE_MAJOR_VERSION}.${PADDLE_MINOR_VERSION}.${PADDLE_PATCH_VERSION}) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake") set(PROJ_ROOT ${CMAKE_SOURCE_DIR}) @@ -64,31 +60,31 @@ include(check_packages) include(swig) include(coveralls) -# add PaddlePaddle version -if(DEFINED ENV{PADDLE_VERSION}) - add_definitions(-DPADDLE_VERSION=\"$ENV{PADDLE_VERSION}\") +# Set PaddlePaddle version to Git tag name or Git commit ID. +find_package(Git REQUIRED) +execute_process( + COMMAND ${GIT_EXECUTABLE} describe --tags --exact-match --abbrev=0 + WORKING_DIRECTORY ${PROJ_ROOT} + OUTPUT_VARIABLE GIT_TAG_NAME + RESULT_VARIABLE GIT_RESULT + ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) +if(NOT ${GIT_RESULT}) + set(PADDLE_VERSION ${GIT_TAG_NAME}) else() - if(EXISTS ${PROJ_ROOT}/.svn/) - find_package(Subversion REQUIRED) - if(SUBVERSION_FOUND) - Subversion_WC_INFO(${PROJ_ROOT} Project) - add_definitions(-DPADDLE_VERSION=${Project_WC_REVISION}) - endif() - elseif(EXISTS ${PROJ_ROOT}/.git/) - find_package(Git REQUIRED) - execute_process( - COMMAND ${GIT_EXECUTABLE} log -1 --format=%H - WORKING_DIRECTORY ${PROJ_ROOT} - OUTPUT_VARIABLE GIT_SHA1 - RESULT_VARIABLE GIT_RESULT - ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) - if(NOT ${GIT_RESULT}) - add_definitions(-DPADDLE_VERSION=\"${GIT_SHA1}\") - else() - message(WARNING "Cannot add paddle version from git tag") - endif() - endif() + execute_process( + COMMAND ${GIT_EXECUTABLE} log -1 --format=%H + WORKING_DIRECTORY ${PROJ_ROOT} + OUTPUT_VARIABLE GIT_SHA1 + RESULT_VARIABLE GIT_RESULT + ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) + if(NOT ${GIT_RESULT}) + set(PADDLE_VERSION ${GIT_SHA1}) + else() + set(PADDLE_VERSION "unknown") + message(WARNING "Cannot add paddle version from git tag") + endif() endif() +add_definitions(-DPADDLE_VERSION=\"${PADDLE_VERSION}\") if(NOT WITH_GPU) From 88802d15cd2a5953efba978040efca86242ba1a2 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Fri, 25 Nov 2016 22:10:38 -0800 Subject: [PATCH 3/7] Remove template execution results from Dockerfile.m4 --- paddle/scripts/docker/Dockerfile.cpu | 12 ------------ paddle/scripts/docker/Dockerfile.cpu-demo | 12 ------------ paddle/scripts/docker/Dockerfile.cpu-devel | 12 ------------ paddle/scripts/docker/Dockerfile.cpu-noavx | 12 ------------ paddle/scripts/docker/Dockerfile.cpu-noavx-demo | 12 ------------ paddle/scripts/docker/Dockerfile.cpu-noavx-devel | 12 ------------ paddle/scripts/docker/Dockerfile.gpu | 12 ------------ paddle/scripts/docker/Dockerfile.gpu-demo | 12 ------------ paddle/scripts/docker/Dockerfile.gpu-devel | 12 ------------ paddle/scripts/docker/Dockerfile.gpu-noavx | 12 ------------ paddle/scripts/docker/Dockerfile.gpu-noavx-demo | 12 ------------ paddle/scripts/docker/Dockerfile.gpu-noavx-devel | 12 ------------ 12 files changed, 144 deletions(-) delete mode 100644 paddle/scripts/docker/Dockerfile.cpu delete mode 100644 paddle/scripts/docker/Dockerfile.cpu-demo delete mode 100644 paddle/scripts/docker/Dockerfile.cpu-devel delete mode 100644 paddle/scripts/docker/Dockerfile.cpu-noavx delete mode 100644 paddle/scripts/docker/Dockerfile.cpu-noavx-demo delete mode 100644 paddle/scripts/docker/Dockerfile.cpu-noavx-devel delete mode 100644 paddle/scripts/docker/Dockerfile.gpu delete mode 100644 paddle/scripts/docker/Dockerfile.gpu-demo delete mode 100644 paddle/scripts/docker/Dockerfile.gpu-devel delete mode 100644 paddle/scripts/docker/Dockerfile.gpu-noavx delete mode 100644 paddle/scripts/docker/Dockerfile.gpu-noavx-demo delete mode 100644 paddle/scripts/docker/Dockerfile.gpu-noavx-devel diff --git a/paddle/scripts/docker/Dockerfile.cpu b/paddle/scripts/docker/Dockerfile.cpu deleted file mode 100644 index 69b8363b7a..0000000000 --- a/paddle/scripts/docker/Dockerfile.cpu +++ /dev/null @@ -1,12 +0,0 @@ -FROM ubuntu:14.04 -MAINTAINER PaddlePaddle Dev Team -COPY build.sh /root/ -ENV GIT_CHECKOUT=v0.9.0a0 -ENV WITH_GPU=OFF -ENV IS_DEVEL=OFF -ENV WITH_DEMO=OFF -ENV PIP_INSTALL_ARGS "" -ENV PIP_GENERAL_ARGS "" -ENV USE_UBUNTU_MIRROR OFF -ENV WITH_AVX=ON -RUN cd /root/ && bash build.sh diff --git a/paddle/scripts/docker/Dockerfile.cpu-demo b/paddle/scripts/docker/Dockerfile.cpu-demo deleted file mode 100644 index ccbd183ee3..0000000000 --- a/paddle/scripts/docker/Dockerfile.cpu-demo +++ /dev/null @@ -1,12 +0,0 @@ -FROM ubuntu:14.04 -MAINTAINER PaddlePaddle Dev Team -COPY build.sh /root/ -ENV GIT_CHECKOUT=v0.9.0a0 -ENV WITH_GPU=OFF -ENV IS_DEVEL=ON -ENV WITH_DEMO=ON -ENV PIP_INSTALL_ARGS "" -ENV PIP_GENERAL_ARGS "" -ENV USE_UBUNTU_MIRROR OFF -ENV WITH_AVX=ON -RUN cd /root/ && bash build.sh diff --git a/paddle/scripts/docker/Dockerfile.cpu-devel b/paddle/scripts/docker/Dockerfile.cpu-devel deleted file mode 100644 index 36460384f3..0000000000 --- a/paddle/scripts/docker/Dockerfile.cpu-devel +++ /dev/null @@ -1,12 +0,0 @@ -FROM ubuntu:14.04 -MAINTAINER PaddlePaddle Dev Team -COPY build.sh /root/ -ENV GIT_CHECKOUT=v0.9.0a0 -ENV WITH_GPU=OFF -ENV IS_DEVEL=ON -ENV WITH_DEMO=OFF -ENV PIP_INSTALL_ARGS "" -ENV PIP_GENERAL_ARGS "" -ENV USE_UBUNTU_MIRROR OFF -ENV WITH_AVX=ON -RUN cd /root/ && bash build.sh diff --git a/paddle/scripts/docker/Dockerfile.cpu-noavx b/paddle/scripts/docker/Dockerfile.cpu-noavx deleted file mode 100644 index fa3b7427b0..0000000000 --- a/paddle/scripts/docker/Dockerfile.cpu-noavx +++ /dev/null @@ -1,12 +0,0 @@ -FROM ubuntu:14.04 -MAINTAINER PaddlePaddle Dev Team -COPY build.sh /root/ -ENV GIT_CHECKOUT=v0.9.0a0 -ENV WITH_GPU=OFF -ENV IS_DEVEL=OFF -ENV WITH_DEMO=OFF -ENV PIP_INSTALL_ARGS "" -ENV PIP_GENERAL_ARGS "" -ENV USE_UBUNTU_MIRROR OFF -ENV WITH_AVX=OFF -RUN cd /root/ && bash build.sh diff --git a/paddle/scripts/docker/Dockerfile.cpu-noavx-demo b/paddle/scripts/docker/Dockerfile.cpu-noavx-demo deleted file mode 100644 index 61315f762d..0000000000 --- a/paddle/scripts/docker/Dockerfile.cpu-noavx-demo +++ /dev/null @@ -1,12 +0,0 @@ -FROM ubuntu:14.04 -MAINTAINER PaddlePaddle Dev Team -COPY build.sh /root/ -ENV GIT_CHECKOUT=v0.9.0a0 -ENV WITH_GPU=OFF -ENV IS_DEVEL=ON -ENV WITH_DEMO=ON -ENV PIP_INSTALL_ARGS "" -ENV PIP_GENERAL_ARGS "" -ENV USE_UBUNTU_MIRROR OFF -ENV WITH_AVX=OFF -RUN cd /root/ && bash build.sh diff --git a/paddle/scripts/docker/Dockerfile.cpu-noavx-devel b/paddle/scripts/docker/Dockerfile.cpu-noavx-devel deleted file mode 100644 index 7636531199..0000000000 --- a/paddle/scripts/docker/Dockerfile.cpu-noavx-devel +++ /dev/null @@ -1,12 +0,0 @@ -FROM ubuntu:14.04 -MAINTAINER PaddlePaddle Dev Team -COPY build.sh /root/ -ENV GIT_CHECKOUT=v0.9.0a0 -ENV WITH_GPU=OFF -ENV IS_DEVEL=ON -ENV WITH_DEMO=OFF -ENV PIP_INSTALL_ARGS "" -ENV PIP_GENERAL_ARGS "" -ENV USE_UBUNTU_MIRROR OFF -ENV WITH_AVX=OFF -RUN cd /root/ && bash build.sh diff --git a/paddle/scripts/docker/Dockerfile.gpu b/paddle/scripts/docker/Dockerfile.gpu deleted file mode 100644 index 1e023ae281..0000000000 --- a/paddle/scripts/docker/Dockerfile.gpu +++ /dev/null @@ -1,12 +0,0 @@ -FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 -MAINTAINER PaddlePaddle Dev Team -COPY build.sh /root/ -ENV GIT_CHECKOUT=v0.9.0a0 -ENV WITH_GPU=ON -ENV IS_DEVEL=OFF -ENV WITH_DEMO=OFF -ENV PIP_INSTALL_ARGS "" -ENV PIP_GENERAL_ARGS "" -ENV USE_UBUNTU_MIRROR OFF -ENV WITH_AVX=ON -RUN cd /root/ && bash build.sh diff --git a/paddle/scripts/docker/Dockerfile.gpu-demo b/paddle/scripts/docker/Dockerfile.gpu-demo deleted file mode 100644 index 92b0dca402..0000000000 --- a/paddle/scripts/docker/Dockerfile.gpu-demo +++ /dev/null @@ -1,12 +0,0 @@ -FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 -MAINTAINER PaddlePaddle Dev Team -COPY build.sh /root/ -ENV GIT_CHECKOUT=v0.9.0a0 -ENV WITH_GPU=ON -ENV IS_DEVEL=ON -ENV WITH_DEMO=ON -ENV PIP_INSTALL_ARGS "" -ENV PIP_GENERAL_ARGS "" -ENV USE_UBUNTU_MIRROR OFF -ENV WITH_AVX=ON -RUN cd /root/ && bash build.sh diff --git a/paddle/scripts/docker/Dockerfile.gpu-devel b/paddle/scripts/docker/Dockerfile.gpu-devel deleted file mode 100644 index fb6f351fd2..0000000000 --- a/paddle/scripts/docker/Dockerfile.gpu-devel +++ /dev/null @@ -1,12 +0,0 @@ -FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 -MAINTAINER PaddlePaddle Dev Team -COPY build.sh /root/ -ENV GIT_CHECKOUT=v0.9.0a0 -ENV WITH_GPU=ON -ENV IS_DEVEL=ON -ENV WITH_DEMO=OFF -ENV PIP_INSTALL_ARGS "" -ENV PIP_GENERAL_ARGS "" -ENV USE_UBUNTU_MIRROR OFF -ENV WITH_AVX=ON -RUN cd /root/ && bash build.sh diff --git a/paddle/scripts/docker/Dockerfile.gpu-noavx b/paddle/scripts/docker/Dockerfile.gpu-noavx deleted file mode 100644 index 7567e62025..0000000000 --- a/paddle/scripts/docker/Dockerfile.gpu-noavx +++ /dev/null @@ -1,12 +0,0 @@ -FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 -MAINTAINER PaddlePaddle Dev Team -COPY build.sh /root/ -ENV GIT_CHECKOUT=v0.9.0a0 -ENV WITH_GPU=ON -ENV IS_DEVEL=OFF -ENV WITH_DEMO=OFF -ENV PIP_INSTALL_ARGS "" -ENV PIP_GENERAL_ARGS "" -ENV USE_UBUNTU_MIRROR OFF -ENV WITH_AVX=OFF -RUN cd /root/ && bash build.sh diff --git a/paddle/scripts/docker/Dockerfile.gpu-noavx-demo b/paddle/scripts/docker/Dockerfile.gpu-noavx-demo deleted file mode 100644 index ac52484c5c..0000000000 --- a/paddle/scripts/docker/Dockerfile.gpu-noavx-demo +++ /dev/null @@ -1,12 +0,0 @@ -FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 -MAINTAINER PaddlePaddle Dev Team -COPY build.sh /root/ -ENV GIT_CHECKOUT=v0.9.0a0 -ENV WITH_GPU=ON -ENV IS_DEVEL=ON -ENV WITH_DEMO=ON -ENV PIP_INSTALL_ARGS "" -ENV PIP_GENERAL_ARGS "" -ENV USE_UBUNTU_MIRROR OFF -ENV WITH_AVX=OFF -RUN cd /root/ && bash build.sh diff --git a/paddle/scripts/docker/Dockerfile.gpu-noavx-devel b/paddle/scripts/docker/Dockerfile.gpu-noavx-devel deleted file mode 100644 index 19202f306b..0000000000 --- a/paddle/scripts/docker/Dockerfile.gpu-noavx-devel +++ /dev/null @@ -1,12 +0,0 @@ -FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04 -MAINTAINER PaddlePaddle Dev Team -COPY build.sh /root/ -ENV GIT_CHECKOUT=v0.9.0a0 -ENV WITH_GPU=ON -ENV IS_DEVEL=ON -ENV WITH_DEMO=OFF -ENV PIP_INSTALL_ARGS "" -ENV PIP_GENERAL_ARGS "" -ENV USE_UBUNTU_MIRROR OFF -ENV WITH_AVX=OFF -RUN cd /root/ && bash build.sh From 74bcf34f2db8e594731bbd613fb50ce2d69e3bed Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Fri, 25 Nov 2016 23:10:08 -0800 Subject: [PATCH 4/7] Update document --- doc/build/docker_install.rst | 129 +++++++++++------------------------ 1 file changed, 41 insertions(+), 88 deletions(-) diff --git a/doc/build/docker_install.rst b/doc/build/docker_install.rst index e95de35f4d..93e6f6dd7e 100644 --- a/doc/build/docker_install.rst +++ b/doc/build/docker_install.rst @@ -1,71 +1,39 @@ -Docker installation guide -========================== +Using and Building Docker Images +================================ -PaddlePaddle provide the `Docker `_ image. `Docker`_ is a lightweight container utilities. The performance of PaddlePaddle in `Docker`_ container is basically as same as run it in a normal linux. The `Docker`_ is a very convenient way to deliver the binary release for linux programs. +We release PaddlePaddle in the form of `Docker `_ images on `dockerhub.com `_. Running as Docker containers is currently the only officially-supported way to running PaddlePaddle. -.. note:: +Run Docker images +----------------- - The `Docker`_ image is the recommended way to run PaddlePaddle +For each version of PaddlePaddle, we release 4 variants of Docker images: -PaddlePaddle Docker images --------------------------- ++-----------------+-------------+-------+ +| | CPU AVX | GPU | ++=================+=============+=======+ +| cpu | yes | no | ++-----------------+-------------+-------+ +| cpu-noavx | no | no | ++-----------------+-------------+-------+ +| gpu | yes | yes | ++-----------------+-------------+-------+ +| gpu-noavx | no | yes | ++-----------------+-------------+-------+ -There are 12 `images `_ for PaddlePaddle, and the name is :code:`paddle-dev/paddle`, tags are\: - - -+-----------------+------------------+------------------------+-----------------------+ -| | normal | devel | demo | -+=================+==================+========================+=======================+ -| CPU | cpu-latest | cpu-devel-latest | cpu-demo-latest | -+-----------------+------------------+------------------------+-----------------------+ -| GPU | gpu-latest | gpu-devel-latest | gpu-demo-latest | -+-----------------+------------------+------------------------+-----------------------+ -| CPU WITHOUT AVX | cpu-noavx-latest | cpu-devel-noavx-latest | cpu-demo-noavx-latest | -+-----------------+------------------+------------------------+-----------------------+ -| GPU WITHOUT AVX | gpu-noavx-latest | gpu-devel-noavx-latest | gpu-demo-noavx-latest | -+-----------------+------------------+------------------------+-----------------------+ - -And the three columns are: - -* normal\: The docker image only contains binary of PaddlePaddle. -* devel\: The docker image contains PaddlePaddle binary, source code and essential build environment. -* demo\: The docker image contains the dependencies to run PaddlePaddle demo. - -And the four rows are: - -* CPU\: CPU Version. Support CPU which has :code:`AVX` instructions. -* GPU\: GPU Version. Support GPU, and cpu has :code:`AVX` instructions. -* CPU WITHOUT AVX\: CPU Version, which support most CPU even doesn't have :code:`AVX` instructions. -* GPU WITHOUT AVX\: GPU Version, which support most CPU even doesn't have :code:`AVX` instructions. - -User can choose any version depends on machine. The following script can help you to detect your CPU support :code:`AVX` or not. +The following command line detects if your CPU supports :code:`AVX`. .. code-block:: bash - + if cat /proc/cpuinfo | grep -q avx ; then echo "Support AVX"; else echo "Not support AVX"; fi -If the output is :code:`Support AVX`, then you can choose the AVX version of PaddlePaddle, otherwise, you need select :code:`noavx` version of PaddlePaddle. For example, the CPU develop version of PaddlePaddle is :code:`paddle-dev/paddle:cpu-devel-latest`. -The PaddlePaddle images don't contain any entry command. You need to write your entry command to use this image. See :code:`Remote Access` part or just use following command to run a :code:`bash` +Once we determine the proper variant, we can cope with the Docker image tag name by appending the version number. For example, the following command runs the AVX-enabled image of the most recent version: .. code-block:: bash - docker run -it paddledev/paddle:cpu-latest /bin/bash + docker run -it --rm paddledev/paddle:cpu-latest /bin/bash - -Download and Run Docker images ------------------------------- - -You have to install Docker in your machine which has linux kernel version 3.10+ first. You can refer to the official guide https://docs.docker.com/engine/installation/ for further information. - -You can use :code:`docker pull ` to download images first, or just launch a container with :code:`docker run` \: - -.. code-block:: bash - - docker run -it paddledev/paddle:cpu-latest - - -If you want to launch container with GPU support, you need to set some environment variables at the same time: +To run a GPU-enabled image, you need to install CUDA and let Docker knows about it: .. code-block:: bash @@ -73,50 +41,35 @@ If you want to launch container with GPU support, you need to set some environme export DEVICES=$(\ls /dev/nvidia* | xargs -I{} echo '--device {}:{}') docker run ${CUDA_SO} ${DEVICES} -it paddledev/paddle:gpu-latest +The default entry point of all our Docker images starts the OpenSSH server. To run PaddlePaddle and to expose OpenSSH port to 2202 on the host computer: -Some notes for docker ---------------------- - -Performance -+++++++++++ - -Since Docker is based on the lightweight virtual containers, the CPU computing performance maintains well. And GPU driver and equipments are all mapped to the container, so the GPU computing performance would not be seriously affected. - -If you use high performance nic, such as RDMA(RoCE 40GbE or IB 56GbE), Ethernet(10GbE), it is recommended to use config "-net = host". - - - +.. code-block:: bash -Remote access -+++++++++++++ + docker run -d -p 2202:22 paddledev/paddle:cpu-latest +Then we can login to the container using username :code:`root` and password :code:`root`: -If you want to enable ssh access background, you need to build an image by yourself. Please refer to official guide https://docs.docker.com/engine/reference/builder/ for further information. +.. code-block:: bash -Following is a simple Dockerfile with ssh: + ssh -p 2202 root@localhost -.. literalinclude:: ../../doc_cn/build_and_install/install/paddle_ssh.Dockerfile -Then you can build an image with Dockerfile and launch a container: +Build Docker images +------------------- -.. code-block:: bash +Developers might want to build Docker images from their local commit or from a tagged version. Suppose that your local repo is at :code:`~/work/Paddle`, the following steps builds a cpu variant from your current work: - # cd into Dockerfile directory - docker build . -t paddle_ssh - # run container, and map host machine port 8022 to container port 22 - docker run -d -p 8022:22 --name paddle_ssh_machine paddle_ssh +.. code-block:: bash -Now, you can ssh on port 8022 to access the container, username is root, password is also root: + cd ~/Paddle + ./paddle/scripts/docker/generates.sh # Use m4 to generate Dockerfiles for each variant. + docker build -t paddle:latest -f ./paddle/scripts/docker/Dockerfile.cpu -.. code-block:: bash +As a release engineer, you might want to build Docker images for a certain version and publish them to dockerhub.com. You can do this by switching to the right Git tag, or create a new tag, before running `docker build`. For example, the following commands build Docker images for v0.9.0: - ssh -p 8022 root@YOUR_HOST_MACHINE - -You can stop and delete the container as following: - -.. code-block:: bash +.. code-block:: bash - # stop - docker stop paddle_ssh_machine - # delete - docker rm paddle_ssh_machine + cd ~/Paddle + git checkout tags/v0.9.0 + ./paddle/scripts/docker/generates.sh # Use m4 to generate Dockerfiles for each variant. + docker build -t paddle:cpu-v0.9.0 -f ./paddle/scripts/docker/Dockerfile.cpu From a3a7e76c7fdce9a75d6e6b5a57c3f189b05bd3b7 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Sat, 26 Nov 2016 20:38:56 -0800 Subject: [PATCH 5/7] Respond to Helin's comments --- doc/build/docker_install.rst | 20 +++++++++++++------- paddle/scripts/docker/Dockerfile.m4 | 3 ++- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/doc/build/docker_install.rst b/doc/build/docker_install.rst index 93e6f6dd7e..5f272aabd7 100644 --- a/doc/build/docker_install.rst +++ b/doc/build/docker_install.rst @@ -20,22 +20,28 @@ For each version of PaddlePaddle, we release 4 variants of Docker images: | gpu-noavx | no | yes | +-----------------+-------------+-------+ -The following command line detects if your CPU supports :code:`AVX`. +We run the following command on Linux to check if the CPU supports :code:`AVX`. -.. code-block:: bash +.. code-block:: bash + + if cat /proc/cpuinfo | grep -i avx; then echo Yes; else echo No; fi + +On Mac OS X, we need to run + +.. code-block:: bash - if cat /proc/cpuinfo | grep -q avx ; then echo "Support AVX"; else echo "Not support AVX"; fi + sysctl -a | grep machdep.cpu.leaf7_features Once we determine the proper variant, we can cope with the Docker image tag name by appending the version number. For example, the following command runs the AVX-enabled image of the most recent version: -.. code-block:: bash +.. code-block:: bash docker run -it --rm paddledev/paddle:cpu-latest /bin/bash To run a GPU-enabled image, you need to install CUDA and let Docker knows about it: -.. code-block:: bash +.. code-block:: bash export CUDA_SO="$(\ls /usr/lib64/libcuda* | xargs -I{} echo '-v {}:{}') $(\ls /usr/lib64/libnvidia* | xargs -I{} echo '-v {}:{}')" export DEVICES=$(\ls /dev/nvidia* | xargs -I{} echo '--device {}:{}') @@ -43,13 +49,13 @@ To run a GPU-enabled image, you need to install CUDA and let Docker knows about The default entry point of all our Docker images starts the OpenSSH server. To run PaddlePaddle and to expose OpenSSH port to 2202 on the host computer: -.. code-block:: bash +.. code-block:: bash docker run -d -p 2202:22 paddledev/paddle:cpu-latest Then we can login to the container using username :code:`root` and password :code:`root`: -.. code-block:: bash +.. code-block:: bash ssh -p 2202 root@localhost diff --git a/paddle/scripts/docker/Dockerfile.m4 b/paddle/scripts/docker/Dockerfile.m4 index 7f08e66653..f2822acdde 100644 --- a/paddle/scripts/docker/Dockerfile.m4 +++ b/paddle/scripts/docker/Dockerfile.m4 @@ -4,7 +4,8 @@ MAINTAINER PaddlePaddle Dev Team # It is good to run apt-get install with Dockerfile RUN directive, # because if the following invocation to /root/build.sh fails, `docker # build` wouldn't have to re-install packages after we fix -# /root/build.sh. +# /root/build.sh. For more about Docker build cache, please refer to +# https://docs.docker.com/engine/userguide/eng-image/dockerfile_best-practices/#/build-cache. RUN apt-get update && \ apt-get install -y cmake libprotobuf-dev protobuf-compiler git \ libgoogle-glog-dev libgflags-dev libatlas-dev libatlas3-base g++ m4 python-pip \ From 08148beebf78a0e759aa9d4d6c095a53820995dd Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Wed, 30 Nov 2016 18:06:32 -0800 Subject: [PATCH 6/7] Use short Git commit id as the version in wheel filename if there is no Git tag --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8e39efcad7..4ac9f70ab1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -83,7 +83,7 @@ if(NOT ${GIT_RESULT}) set(PADDLE_VERSION ${GIT_TAG_NAME}) else() execute_process( - COMMAND ${GIT_EXECUTABLE} log -1 --format=%H + COMMAND ${GIT_EXECUTABLE} log -1 --format=%h WORKING_DIRECTORY ${PROJ_ROOT} OUTPUT_VARIABLE GIT_SHA1 RESULT_VARIABLE GIT_RESULT From a514eb395aea58cb6863caf61ba163bbb4f3cb38 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Wed, 30 Nov 2016 22:56:45 +0800 Subject: [PATCH 7/7] Refine cmake get version routine --- CMakeLists.txt | 24 ++---------------------- cmake/version.cmake | 24 ++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 22 deletions(-) create mode 100644 cmake/version.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 4ac9f70ab1..7b42423749 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -73,28 +73,8 @@ include(coveralls) # Set PaddlePaddle version to Git tag name or Git commit ID. find_package(Git REQUIRED) -execute_process( - COMMAND ${GIT_EXECUTABLE} describe --tags --exact-match --abbrev=0 - WORKING_DIRECTORY ${PROJ_ROOT} - OUTPUT_VARIABLE GIT_TAG_NAME - RESULT_VARIABLE GIT_RESULT - ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) -if(NOT ${GIT_RESULT}) - set(PADDLE_VERSION ${GIT_TAG_NAME}) -else() - execute_process( - COMMAND ${GIT_EXECUTABLE} log -1 --format=%h - WORKING_DIRECTORY ${PROJ_ROOT} - OUTPUT_VARIABLE GIT_SHA1 - RESULT_VARIABLE GIT_RESULT - ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) - if(NOT ${GIT_RESULT}) - set(PADDLE_VERSION ${GIT_SHA1}) - else() - set(PADDLE_VERSION "unknown") - message(WARNING "Cannot add paddle version from git tag") - endif() -endif() +# version.cmake will get the current PADDLE_VERSION +include(version) add_definitions(-DPADDLE_VERSION=\"${PADDLE_VERSION}\") diff --git a/cmake/version.cmake b/cmake/version.cmake new file mode 100644 index 0000000000..a0518e07e8 --- /dev/null +++ b/cmake/version.cmake @@ -0,0 +1,24 @@ +# Get the latest git tag. +set(PADDLE_VERSION $ENV{PADDLE_VERSION}) +set(tmp_version "HEAD") +while ("${PADDLE_VERSION}" STREQUAL "") + execute_process( + COMMAND ${GIT_EXECUTABLE} describe --tags --abbrev=0 ${tmp_version} + WORKING_DIRECTORY ${PROJ_ROOT} + OUTPUT_VARIABLE GIT_TAG_NAME + RESULT_VARIABLE GIT_RESULT + ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) + if (NOT ${GIT_RESULT}) + # Check the tag is a correct version + if (${GIT_TAG_NAME} MATCHES "v[0-9]+\\.[0-9]+\\.[0-9]+(\\.(a|b|rc)\\.[0-9]+)?") + string(REPLACE "v" "" PADDLE_VERSION ${GIT_TAG_NAME}) + else() # otherwise, get the previous git tag name. + set(tmp_version "${GIT_TAG_NAME}~1") + endif() + else() + set(PADDLE_VERSION "0.0.0") + message(WARNING "Cannot add paddle version from git tag") + endif() +endwhile() + +message(STATUS "Paddle version is ${PADDLE_VERSION}")