add musl docker build script (#28027)

* add musl docker build script * rm space test=document_fix * fix some docs and types errors test=document_fix
5 years ago · 953302d9eb
parent 17db031a05
commit 953302d9eb
6 changed files with 344 additions and 0 deletions
--- a/paddle/scripts/musl_build/Dockerfile
+++ b/paddle/scripts/musl_build/Dockerfile
@ -0,0 +1,15 @@
+FROM python:3.7-alpine3.10
+
+WORKDIR /root
+
+RUN apk update
+
+RUN apk add --no-cache \
+    g++  gfortran make cmake patchelf git \
+    linux-headers \
+    freetype-dev libjpeg-turbo-dev zlib-dev
+
+RUN apk add --no-cache --force-overwrite \
+    lapack-dev openblas-dev 
+
+ENTRYPOINT [ "/bin/sh" ]
--- a/paddle/scripts/musl_build/README.md
+++ b/paddle/scripts/musl_build/README.md
@ -0,0 +1,90 @@
+Paddle for Linux-musl Usage Guide
+===========================================
+
+# introduction
+Paddle can be built for linux-musl such as alpine, and be used in libos-liked SGX TEE environment. Currently supported commericial product TEE Scone, and community maintanced TEE Occlum. We also working on to support open source TEE Graphene.
+
+
+# build automaticly
+1. clone paddle source from github
+   
+```bash
+git clone https://github.com/PaddlePaddle/Paddle.git
+```
+
+2. setup build directory
+
+```bash
+# enter paddle directory
+cd  ./Paddle
+
+# create and enter building directory
+mkdir -p build && cd build
+```
+
+3. build docker for compiling. use environment HTTP_PROXY/HTTPS_PROXY for proxy setup.
+
+```bash
+# setup proxy address
+export HTTP_PROXY='http://127.0.0.1:8080'
+export HTTPS_PROXY='https://127.0.0.1:8080'
+
+# invoke build script
+../paddle/scripts/musl_build/build_docker.sh
+```
+
+4. compile paddle in previous built docker. proxy setup method is same as previous step.
+output wheel package will save to "dist" directory.
+
+```bash
+# setup proxy addresss
+export HTTP_PROXY='http://127.0.0.1:8080'
+export HTTPS_PROXY='https://127.0.0.1:8080'
+
+# invoke build paddle script
+../paddle/scripts/musl_build/build_paddle.sh
+
+# find output wheel package
+ls dist/*.whl
+```
+
+# build paddle manually  
+
+1. start up the building docker, and enter the shell in the container
+```bash
+# checkout paddle source code
+git clone https://github.com/PaddlePaddle/Paddle.git
+
+# entery paddle directory
+cd ./Paddle
+
+# build docker image
+../paddle/scripts/musl_build/build_docker.sh
+
+# enter the container interactive shell
+BUILD_AUTO=0 ../paddle/scripts/musl_build/build_paddle.sh
+```
+
+2. Type commands to compile source manually
+```sh
+# compile paddle by commands
+# paddle is mount to /paddle directory
+# working directory is /root
+mkdir build && cd build
+
+# install python requirement
+pip install -r /paddle/python/requirements.txt
+
+# configure project with cmake
+cmake /paddle -DWITH_MUSL=ON DWITH_CRYPTO=OFF -DWITH_MKL=OFF -DWITH_GPU=OFF -DWITH_TESTING=OFF
+
+# run the make to build project
+make
+```
+
+# files
+- build_docker.sh: docker building script
+- build_paddle.sh: paddle building script
+- build_inside.sh: build_paddle.sh will invoke this script inside the docker for compiling.
+- config.sh: build config script for configure compiling option setting.
+- Dockerfile: build docker defination file.
--- a/paddle/scripts/musl_build/build_docker.sh
+++ b/paddle/scripts/musl_build/build_docker.sh
@ -0,0 +1,50 @@
+#!/bin/bash
+
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+CUR_DIR=$(dirname "${BASH_SOURCE[0]}")
+CUR_DIR=$(realpath "$CUR_DIR")
+
+# shellcheck disable=1090
+source "$CUR_DIR/config.sh"
+
+# exit when any command fails
+set -e
+
+declare -a ENV_ARGS
+if [ "$HTTP_PROXY" ]; then
+    ENV_ARGS+=("--build-arg" "http_proxy=$HTTP_PROXY")
+    echo "using http proxy: $HTTP_PROXY"
+fi
+
+if [ "$HTTPS_PROXY" ]; then
+    ENV_ARGS+=("--build-arg" "https_proxy=$HTTPS_PROXY")
+    echo "using https proxy: $HTTPS_PROXY"
+fi
+
+echo "clean up docker images: $BUILD_IMAGE"
+docker rmi -f "$BUILD_IMAGE"
+
+echo "build docker image: $BUILD_IMAGE"
+
+# shellcheck disable=2086
+docker build \
+    -t "$BUILD_IMAGE" \
+    -f "$CUR_DIR/Dockerfile" \
+    --rm=false \
+    --network host \
+    ${ENV_ARGS[*]} \
+    --output type=tar,dest=build.tar \
+    .
--- a/paddle/scripts/musl_build/build_inside.sh
+++ b/paddle/scripts/musl_build/build_inside.sh
@ -0,0 +1,64 @@
+#!/bin/sh
+
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+PADDLE_DIR=/paddle
+BUILD_DIR=$PWD
+
+echo "paddle: $PADDLE_DIR"
+echo "python: $PYTHON_VERSION"
+echo "http_proxy: $HTTP_PROXY"
+echo "https_proxy: $HTTPS_PROXY"
+
+# exit when any command fails
+set -e
+
+echo "create build dir: $BUILD_DIR"
+mkdir -p "$BUILD_DIR"
+
+if [ "$HTTP_PROXY" ]; then
+    git config --global http.proxy "$HTTP_PROXY"
+fi
+
+if [ "$HTTP_PROXY" ]; then
+    git config --global https.proxy "$HTTPS_PROXY"
+fi
+
+PIP_ARGS=""
+if [ "$PIP_INDEX" ]; then
+    PIP_DOMAIN=$(echo "$PIP_INDEX" | awk -F/ '{print $3}')
+    PIP_ARGS="-i $PIP_INDEX --trusted-host $PIP_DOMAIN"
+    echo "pip index: $PIP_INDEX"
+fi
+
+PYTHON_REQS=$PADDLE_DIR/python/requirements.txt
+echo "install python requirements: $PYTHON_REQS"
+
+# shellcheck disable=2086
+pip install $PIP_ARGS --timeout 300 --no-cache-dir -r $PYTHON_REQS
+
+echo "configure with cmake"
+cmake "$PADDLE_DIR" \
+    -DWITH_MUSL=ON \
+    -DWITH_CRYPTO=OFF \
+    -DWITH_MKL=OFF \
+    -DWITH_GPU=OFF
+
+echo "compile with make: $*"
+# shellcheck disable=2068
+make $@
+
+echo "save python dist directory to /output"
+cp -r python/dist /output/
--- a/paddle/scripts/musl_build/build_paddle.sh
+++ b/paddle/scripts/musl_build/build_paddle.sh
@ -0,0 +1,101 @@
+#!/bin/bash
+
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+CUR_DIR=$(dirname "${BASH_SOURCE[0]}")
+CUR_DIR=$(realpath "$CUR_DIR")
+
+# shellcheck disable=1090
+source "$CUR_DIR/config.sh"
+
+# exit when any command fails
+set -e
+
+# check build mode auto/man
+BUILD_AUTO=${BUILD_AUTO:-1}
+
+
+declare -a ENV_ARGS
+if [ "$HTTP_PROXY" ]; then
+    ENV_ARGS+=("--env" "HTTP_PROXY=$HTTP_PROXY")
+    echo "using http proxy: $HTTP_PROXY"
+fi
+
+if [ "$HTTPS_PROXY" ]; then
+    ENV_ARGS+=("--env" "HTTPS_PROXY=$HTTPS_PROXY")
+    echo "using https proxy: $HTTPS_PROXY"
+fi
+
+if [ "$PIP_INDEX" ]; then
+    ENV_ARGS+=("--env" "PIP_INDEX=$PIP_INDEX")
+fi
+
+echo "compile paddle in docker"
+echo "docker image: $BUILD_IMAGE"
+
+BUILD_ID=$(docker images -q "$BUILD_IMAGE")
+if [ ! "$BUILD_ID" ]; then
+    echo "docker image is not existed, and try to build."
+
+    "$CUR_DIR/build_docker.sh"
+fi
+
+BUILD_NAME="paddle-musl-build-$(date +%Y%m%d-%H%M%S)"
+echo "container name: $BUILD_NAME"
+
+MOUNT_DIR="/paddle"
+echo "mount paddle: $PADDLE_DIR => $MOUNT_DIR"
+
+
+if [ "$BUILD_AUTO" -eq "1" ]; then
+    echo "enter automatic build mode"
+
+    # no exit when fails
+    set +e
+
+    BUILD_SCRIPT=$MOUNT_DIR/paddle/scripts/musl_build/build_inside.sh
+    echo "build script: $BUILD_SCRIPT"
+
+    OUTPUT_DIR="output"
+    mkdir -p $OUTPUT_DIR
+    OUTPUT_DIR=$(realpath $OUTPUT_DIR)
+    echo "build output: $OUTPUT_DIR"
+
+    # shellcheck disable=2086,2068
+    docker run \
+        -v "$PADDLE_DIR":"$MOUNT_DIR" \
+        -v "$OUTPUT_DIR":/output \
+        --rm \
+        --workdir /root \
+        --network host \
+        ${ENV_ARGS[*]} \
+        --name "$BUILD_NAME" \
+        "$BUILD_IMAGE" \
+        "$BUILD_SCRIPT" $@
+
+    echo "list output: $OUTPUT_DIR"
+    ls "$OUTPUT_DIR"
+else
+    echo "enter manual build mode"
+
+    # shellcheck disable=2086
+    docker run \
+        -it \
+        -v "$PADDLE_DIR":"$MOUNT_DIR" \
+        --workdir /root \
+        --network host ${ENV_ARGS[*]}\
+        --name "$BUILD_NAME" \
+        "$BUILD_IMAGE"
+fi
--- a/paddle/scripts/musl_build/config.sh
+++ b/paddle/scripts/musl_build/config.sh
@ -0,0 +1,24 @@
+#!/bin/bash
+
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+CUR_DIR=$(dirname "${BASH_SOURCE[0]}")
+CUR_DIR=$(realpath "$CUR_DIR")
+
+# shellcheck disable=2034
+PADDLE_DIR=$(realpath "$CUR_DIR/../../../")
+
+# shellcheck disable=2034
+BUILD_IMAGE="paddle-musl-build:2.0"