m53297601
/
Paddle
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

use deb to build paddle-core image

Browse Source
0.10.0rc
yi.wu 8 years ago
parent f66fd44fec
commit 1b269dbe06
7 changed files with 130 additions and 35 deletions
Show Stats Download Patch File Download Diff File

18
paddle/scripts/deb/build_scripts/build.sh
Unescape View File

@ -1,5 +1,6 @@
#!/bin/bash #!/bin/bash
set -e set -e
ARCH=$(uname -i)
apt-get update apt-get update
apt-get install -y dh-make apt-get install -y dh-make
cd ~ cd ~
@ -8,28 +9,35 @@ mkdir -p ~/dist/cpu
mkdir -p ~/dist/cpu-noavx mkdir -p ~/dist/cpu-noavx
mkdir -p ~/dist/gpu-noavx mkdir -p ~/dist/gpu-noavx
cd paddle cd paddle
mkdir build
# clean build dir and third_party dir cache
rm -rf build third_party
mkdir -p build
cd build cd build
cmake .. -DWITH_GPU=OFF -DWITH_SWIG_PY=ON -DWITH_AVX=ON cmake .. -DWITH_GPU=OFF -DWITH_SWIG_PY=ON -DWITH_AVX=ON -DWITH_SWIG_PY=ON -DWITH_STYLE_CHECK=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
make -j `nproc` make -j `nproc`
# FIXME(typhoonzero): add ARCH gpu noavx flag to CPACK_SYSTEM_NAME. Why -D not affect anything?
cpack -D CPACK_GENERATOR='DEB' .. cpack -D CPACK_GENERATOR='DEB' ..
mv *.deb ~/dist/cpu mv *.deb ~/dist/cpu
rm -rf * rm -rf *
cmake .. -DWITH_GPU=ON -DWITH_SWIG_PY=ON -DWITH_AVX=ON -DCUDNN_ROOT=/usr/ ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so /usr/lib/libcudnn.so
cmake .. -DWITH_GPU=ON -DWITH_SWIG_PY=ON -DWITH_AVX=ON -DCUDNN_ROOT=/usr/ -DWITH_SWIG_PY=ON -DWITH_STYLE_CHECK=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
make -j `nproc` make -j `nproc`
cpack -D CPACK_GENERATOR='DEB' .. cpack -D CPACK_GENERATOR='DEB' ..
mv *.deb ~/dist/gpu mv *.deb ~/dist/gpu
rm -rf * rm -rf *
cmake .. -DWITH_GPU=OFF -DWITH_SWIG_PY=ON -DWITH_AVX=OFF rm -f /usr/lib/libcudnn.so
cmake .. -DWITH_GPU=OFF -DWITH_SWIG_PY=ON -DWITH_AVX=OFF -DWITH_SWIG_PY=ON -DWITH_STYLE_CHECK=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
make -j `nproc` make -j `nproc`
cpack -D CPACK_GENERATOR='DEB' .. cpack -D CPACK_GENERATOR='DEB' ..
mv *.deb ~/dist/cpu-noavx mv *.deb ~/dist/cpu-noavx
rm -rf * rm -rf *
cmake .. -DWITH_GPU=ON -DWITH_SWIG_PY=ON -DWITH_AVX=OFF -DCUDNN_ROOT=/usr/ ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so /usr/lib/libcudnn.so
cmake .. -DWITH_GPU=ON -DWITH_SWIG_PY=ON -DWITH_AVX=OFF -DCUDNN_ROOT=/usr/ -DWITH_SWIG_PY=ON -DWITH_STYLE_CHECK=OFF -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
make -j `nproc` make -j `nproc`
cpack -D CPACK_GENERATOR='DEB' .. cpack -D CPACK_GENERATOR='DEB' ..
mv *.deb ~/dist/gpu-noavx mv *.deb ~/dist/gpu-noavx

27
paddle/scripts/docker/buildall.sh
Unescape View File

@ -1,32 +1,31 @@
#!/bin/bash #!/bin/bash
BINARIES_DIR=paddle/scripts/docker/buildimage/binaries
BUILD_DIR=$PWD/build BUILD_DIR=$PWD/build
DEB_DIST_DIR=$PWD/dist
VERSION=latest
function build_in_docker() { function build_in_docker() {
if [ ! -d $BUILD_DIR ]; then if [ ! -d $BUILD_DIR ]; then
mkdir -p $BUILD_DIR mkdir -p $BUILD_DIR
fi fi
if [ ! -d $DEB_DIST_DIR ]; then
mkdir -p $DEB_DIST_DIR
fi
docker build . -t paddle-build-env -f paddle/scripts/docker/paddle-dev/Dockerfile docker build . -t paddle-build-env -f paddle/scripts/docker/paddle-dev/Dockerfile
# FIXME: need to wait a signal not sleeping # FIXME: need to wait a signal not sleeping
BUILDER=$(docker run -d -v ${PWD}:/paddle paddle-build-env sleep 3600) BUILDER=$(docker run -d -v ${PWD}:/root/paddle -v ${DEB_DIST_DIR}:/root/dist paddle-build-env sleep 3600)
# TODO(typhoonzero): # NOTICE: build deb files for real paddle image
docker exec $BUILDER /bin/bash -c "export BUILD_AND_INSTALL=ON && /paddle/paddle/scripts/docker/build.sh" docker exec $BUILDER /bin/bash -c "/root/paddle/paddle/scripts/deb/build_scripts/build.sh"
mkdir -p $BINARIES_DIR
# docker cp $BUILDER:/usr/local/opt/paddle/bin/paddle_pserver_main $BINARIES_DIR
# docker cp $BUILDER:/usr/local/opt/paddle/bin/paddle_trainer $BINARIES_DIR
# docker cp $BUILDER:/usr/local/opt/paddle/bin/paddle_merge_model $BINARIES_DIR
# docker cp $BUILDER:/usr/local/bin/paddle $BINARIES_DIR
# docker cp $BUILDER:/usr/local/opt/paddle/bin/paddle_usage $BINARIES_DIR
#
# docker cp $BUILDER:/usr/local/opt/paddle/share/wheels $BINARIES_DIR
docker stop $BUILDER && docker rm $BUILDER docker stop $BUILDER && docker rm $BUILDER
} }
function build_paddle_core() { function build_paddle_core() {
docker build . -t paddle-core -f paddle/scripts/docker/paddle-core/Dockerfile docker build . -t paddle-core:$VERSION -f paddle/scripts/docker/paddle-core/Dockerfile
docker build . -t paddle-core:gpu-$VERSION -f paddle/scripts/docker/paddle-core/Dockerfile.gpu
docker build . -t paddle-core:cpu-noavx-$VERSION -f paddle/scripts/docker/paddle-core/Dockerfile.noavx
docker build . -t paddle-core:gpu-noavx-$VERSION -f paddle/scripts/docker/paddle-core/Dockerfile.gpunoavx
} }
build_in_docker build_in_docker
#build_paddle_core build_paddle_core

16
paddle/scripts/docker/paddle-core/Dockerfile
Unescape View File

@ -23,20 +23,10 @@ RUN sed 's@http:\/\/archive.ubuntu.com\/ubuntu\/@mirror:\/\/mirrors.ubuntu.com\/
pip install --upgrade pip && \ pip install --upgrade pip && \
pip install -U 'protobuf==3.1.0' pip install -U 'protobuf==3.1.0'
RUN pip install numpy RUN pip install numpy
# Use different deb file when building different type of images
ADD dist/cpu/*.deb /usr/local/opt/paddle/deb/cpu/
RUN dpkg --force-all -i /usr/local/opt/paddle/deb/cpu/*.deb && rm -f /usr/local/opt/paddle/deb/cpu/*.deb
ADD build/paddle/trainer/paddle_trainer \
build/paddle/pserver/paddle_pserver_main \
build/paddle/trainer/paddle_merge_model \
/usr/local/opt/paddle/bin/
ADD build/paddle/scripts/usage.sh /usr/local/opt/paddle/bin/paddle_usage
ADD paddle/scripts/submit_local.sh.in /usr/bin/paddle
ADD paddle/dist/py_paddle*linux*.whl build/python/dist/paddle*.whl \
/usr/local/opt/paddle/share/wheels/
RUN pip install /usr/local/opt/paddle/share/wheels/py_paddle*linux*.whl && \
pip install /usr/local/opt/paddle/share/wheels/paddle*.whl && \
rm /usr/local/opt/paddle/share/wheels/py_paddle*linux*.whl && \
rm /usr/local/opt/paddle/share/wheels/paddle*.whl
ENV PATH="/usr/local/opt/paddle/bin/:${PATH}" ENV PATH="/usr/local/opt/paddle/bin/:${PATH}"
# default command shows the paddle version and exit # default command shows the paddle version and exit
CMD ["paddle", "version"] CMD ["paddle", "version"]

32
paddle/scripts/docker/paddle-core/Dockerfile.gpu
Unescape View File

@ -0,0 +1,32 @@
FROM nvidia/cuda:7.5-cudnn5-runtime-ubuntu14.04
MAINTAINER PaddlePaddle Authors <paddle-dev@baidu.com>
# ENV variables
ARG WITH_AVX
ARG WITH_DOC
ARG WITH_STYLE_CHECK
ENV WITH_GPU=OFF
ENV WITH_AVX=${WITH_AVX:-ON}
ENV WITH_DOC=${WITH_DOC:-OFF}
ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF}
ENV HOME /root
ENV LANG en_US.UTF-8
# Use Fix locales to en_US.UTF-8
RUN sed 's@http:\/\/archive.ubuntu.com\/ubuntu\/@mirror:\/\/mirrors.ubuntu.com\/mirrors.txt@' -i /etc/apt/sources.list && \
apt-get update && \
apt-get install -y python python-pip libgfortran3 && \
apt-get clean -y && \
pip install --upgrade pip && \
pip install -U 'protobuf==3.1.0'
RUN pip install numpy
# Use different deb file when building different type of images
ADD dist/gpu/*.deb /usr/local/opt/paddle/deb/gpu/
RUN dpkg --force-all -i /usr/local/opt/paddle/deb/gpu/*.deb && rm -f /usr/local/opt/paddle/deb/gpu/*.deb
ENV PATH="/usr/local/opt/paddle/bin/:${PATH}"
# default command shows the paddle version and exit
CMD ["paddle", "version"]

33
paddle/scripts/docker/paddle-core/Dockerfile.gpunoavx
Unescape View File

@ -0,0 +1,33 @@
FROM python:2.7.13-slim
MAINTAINER PaddlePaddle Authors <paddle-dev@baidu.com>
# ENV variables
ARG WITH_AVX
ARG WITH_DOC
ARG WITH_STYLE_CHECK
ENV WITH_GPU=OFF
ENV WITH_AVX=${WITH_AVX:-ON}
ENV WITH_DOC=${WITH_DOC:-OFF}
ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF}
ENV HOME /root
ENV LANG en_US.UTF-8
# Use Fix locales to en_US.UTF-8
RUN sed 's@http:\/\/archive.ubuntu.com\/ubuntu\/@mirror:\/\/mirrors.ubuntu.com\/mirrors.txt@' -i /etc/apt/sources.list && \
apt-get update && \
apt-get install -y libgfortran3 && \
apt-get clean -y && \
pip install --upgrade pip && \
pip install -U 'protobuf==3.1.0'
RUN pip install numpy
# Use different deb file when building different type of images
ADD dist/gpu-noavx/*.deb /usr/local/opt/paddle/deb/gpu-noavx/
RUN dpkg --force-all -i /usr/local/opt/paddle/deb/gpu-noavx/*.deb && rm -f /usr/local/opt/paddle/deb/gpu-noavx/*.deb
ENV PATH="/usr/local/opt/paddle/bin/:${PATH}"
# default command shows the paddle version and exit
CMD ["paddle", "version"]

32
paddle/scripts/docker/paddle-core/Dockerfile.noavx
Unescape View File

@ -0,0 +1,32 @@
FROM nvidia/cuda:7.5-cudnn5-runtime-ubuntu14.04
MAINTAINER PaddlePaddle Authors <paddle-dev@baidu.com>
# ENV variables
ARG WITH_AVX
ARG WITH_DOC
ARG WITH_STYLE_CHECK
ENV WITH_GPU=OFF
ENV WITH_AVX=${WITH_AVX:-ON}
ENV WITH_DOC=${WITH_DOC:-OFF}
ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF}
ENV HOME /root
ENV LANG en_US.UTF-8
# Use Fix locales to en_US.UTF-8
RUN sed 's@http:\/\/archive.ubuntu.com\/ubuntu\/@mirror:\/\/mirrors.ubuntu.com\/mirrors.txt@' -i /etc/apt/sources.list && \
apt-get update && \
apt-get install -y python python-pip libgfortran3 && \
apt-get clean -y && \
pip install --upgrade pip && \
pip install -U 'protobuf==3.1.0'
RUN pip install numpy
# Use different deb file when building different type of images
ADD dist/cpu-noavx/*.deb /usr/local/opt/paddle/deb/cpu-noavx/
RUN dpkg --force-all -i /usr/local/opt/paddle/deb/cpu-noavx/*.deb && rm -f /usr/local/opt/paddle/deb/cpu-noavx/*.deb
ENV PATH="/usr/local/opt/paddle/bin/:${PATH}"
# default command shows the paddle version and exit
CMD ["paddle", "version"]

7
paddle/scripts/docker/paddle-dev/Dockerfile
Unescape View File

@ -1,5 +1,6 @@
# A image for building paddle binaries # A image for building paddle binaries
FROM ubuntu:14.04 # Use cuda devel base image for both cpu and gpu environment
FROM nvidia/cuda:7.5-cudnn5-devel-ubuntu14.04
MAINTAINER PaddlePaddle Authors <paddle-dev@baidu.com> MAINTAINER PaddlePaddle Authors <paddle-dev@baidu.com>
ARG DEBIAN_FRONTEND=noninteractive ARG DEBIAN_FRONTEND=noninteractive
@ -22,8 +23,7 @@ ENV WITH_STYLE_CHECK=${WITH_STYLE_CHECK:-OFF}
ENV HOME /root ENV HOME /root
RUN sed 's@http:\/\/archive.ubuntu.com\/ubuntu\/@mirror:\/\/mirrors.ubuntu.com\/mirrors.txt@' -i /etc/apt/sources.list && \ RUN apt-get update && \
apt-get update && \
apt-get install -y git python-pip python-dev openssh-server bison && \ apt-get install -y git python-pip python-dev openssh-server bison && \
apt-get install -y wget unzip tar xz-utils bzip2 gzip coreutils && \ apt-get install -y wget unzip tar xz-utils bzip2 gzip coreutils && \
apt-get install -y curl sed grep graphviz libjpeg-dev zlib1g-dev && \ apt-get install -y curl sed grep graphviz libjpeg-dev zlib1g-dev && \
@ -48,5 +48,6 @@ RUN curl -sSL https://cmake.org/files/v3.4/cmake-3.4.1.tar.gz | tar -xz && \
cd cmake-3.4.1 && ./bootstrap && make -j `nproc` && make install && \ cd cmake-3.4.1 && ./bootstrap && make -j `nproc` && make install && \
cd .. && rm -rf cmake-3.4.1 cd .. && rm -rf cmake-3.4.1
RUN apt-get install -y swig
# FIXME: wait a long time is OK # FIXME: wait a long time is OK
CMD ["sleep", "3600"] CMD ["sleep", "3600"]

Write Preview
Loading…
Cancel
Save