m53297601
/
Paddle
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

add parallel build script to ci … (#16901)

Browse Source 
* add parallel build script to ci test=develop
* 1. classify the test case as single card/two cards/multiple cards type
   2. run test case according to the run type
shanyi15-patch-1
wopeizl 6 years ago committed by GitHub
parent 24923f7604
commit d9991dccdd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 211 additions and 5 deletions
Show Stats Download Patch File Download Diff File

1
paddle/fluid/inference/api/CMakeLists.txt
Unescape View File

@ -56,6 +56,7 @@ if(WITH_TESTING)
inference_base_test(test_api_impl SRCS api_impl_tester.cc DEPS ${inference_deps}
ARGS --word2vec_dirname=${WORD2VEC_MODEL_DIR} --book_dirname=${PYTHON_TESTS_DIR}/book)
set_tests_properties(test_api_impl PROPERTIES DEPENDS test_image_classification)
set_tests_properties(test_api_impl PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE")
endif()
cc_test(test_analysis_predictor SRCS analysis_predictor_tester.cc DEPS analysis_predictor benchmark ${inference_deps}
ARGS --dirname=${WORD2VEC_MODEL_DIR})

5
paddle/fluid/inference/tests/book/CMakeLists.txt
Unescape View File

@ -19,7 +19,9 @@ function(inference_test TARGET_NAME)
DEPS paddle_fluid_origin
ARGS --dirname=${PYTHON_TESTS_DIR}/book/${TARGET_NAME}${arg}.inference.model)
set_tests_properties(test_inference_${TARGET_NAME}${arg}
PROPERTIES DEPENDS test_${TARGET_NAME})
PROPERTIES DEPENDS test_${TARGET_NAME})
set_tests_properties(test_inference_${TARGET_NAME}${arg}
PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE")
endforeach()
endfunction(inference_test)
@ -45,3 +47,4 @@ cc_test(test_inference_nlp
DEPS paddle_fluid_origin
ARGS
--model_path=${PADDLE_BINARY_DIR}/python/paddle/fluid/tests/book/recognize_digits_mlp.inference.model)
set_tests_properties(test_inference_nlp PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE")

2
paddle/fluid/train/CMakeLists.txt
Unescape View File

@ -20,6 +20,8 @@ function(train_test TARGET_NAME)
ARGS --dirname=${PYTHON_TESTS_DIR}/book/${TARGET_NAME}${arg}.train.model/)
set_tests_properties(test_train_${TARGET_NAME}${arg}
PROPERTIES DEPENDS test_${TARGET_NAME})
set_tests_properties(test_train_${TARGET_NAME}${arg}
PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE")
endforeach()
endfunction(train_test)

197
paddle/scripts/paddle_build.sh
Unescape View File

@ -561,6 +561,195 @@ function bind_test() {
wait
}
function parallel_test() {
mkdir -p ${PADDLE_ROOT}/build
cd ${PADDLE_ROOT}/build
if [ ${WITH_TESTING:-ON} == "ON" ] ; then
cat <<EOF
========================================
Running unit tests ...
========================================
EOF
# calculate and set the memory usage for each process
# MEM_USAGE=$(printf "%.2f" `echo "scale=5; 1.0 / $NUM_PROC" | bc`)
# export FLAGS_fraction_of_gpu_memory_to_use=$MEM_USAGE
EXIT_CODE=0;
pids=()
# get the CUDA device count
CUDA_DEVICE_COUNT=$(nvidia-smi -L | wc -l)
# each test case would occupy two graph cards
NUM_PROC=$[CUDA_DEVICE_COUNT/2]
for (( i = 0; i < $NUM_PROC; i++ )); do
# CUDA_VISIBLE_DEVICES http://acceleware.com/blog/cudavisibledevices-masking-gpus
# ctest -I https://cmake.org/cmake/help/v3.0/manual/ctest.1.html?highlight=ctest
if [ ${TESTING_DEBUG_MODE:-OFF} == "ON" ] ; then
env CUDA_VISIBLE_DEVICES=$[i*2],$[i*2+1] ctest -I $i,,$NUM_PROC -V &
pids+=($!)
else
env CUDA_VISIBLE_DEVICES=$[i*2],$[i*2+1] ctest -I $i,,$NUM_PROC --output-on-failure &
pids+=($!)
fi
done
clen=`expr "${#pids[@]}" - 1` # get length of commands - 1
for i in `seq 0 "$clen"`; do
wait ${pids[$i]}
CODE=$?
if [[ "${CODE}" != "0" ]]; then
echo "At least one test failed with exit code => ${CODE}" ;
EXIT_CODE=1;
fi
done
wait; # wait for all subshells to finish
echo "EXIT_CODE => $EXIT_CODE"
if [[ "${EXIT_CODE}" != "0" ]]; then
exit "$EXIT_CODE"
fi
fi
}
EXIT_CODE=0;
function caught_error() {
for job in `jobs -p`; do
# echo "PID => ${job}"
if ! wait ${job} ; then
echo "At least one test failed with exit code => $?" ;
EXIT_CODE=1;
fi
done
}
function card_test() {
set -m
# get the CUDA device count
CUDA_DEVICE_COUNT=$(nvidia-smi -L | wc -l)
testcases=$1
if (( $# > 1 )); then
cardnumber=$2
if (( $cardnumber > $CUDA_DEVICE_COUNT )); then
cardnumber=$CUDA_DEVICE_COUNT
fi
else
cardnumber=$CUDA_DEVICE_COUNT
fi
if [[ "$testcases" == "" ]]; then
return 0
fi
trap 'caught_error' CHLD
NUM_PROC=$[CUDA_DEVICE_COUNT/$cardnumber]
for (( i = 0; i < $NUM_PROC; i++ )); do
# CUDA_VISIBLE_DEVICES http://acceleware.com/blog/cudavisibledevices-masking-gpus
# ctest -I https://cmake.org/cmake/help/v3.0/manual/ctest.1.html?highlight=ctest
cuda_list=()
for (( j = 0; j < cardnumber; j++ )); do
if [ $j -eq 0 ]; then
cuda_list=("$[i*cardnumber]")
else
cuda_list="$cuda_list,$[i*cardnumber+j]"
fi
done
# echo $cuda_list
if [ ${TESTING_DEBUG_MODE:-OFF} == "ON" ] ; then
if [[ $cardnumber == $CUDA_DEVICE_COUNT ]]; then
ctest -I $i,,$NUM_PROC -R "($testcases)" -V &
else
env CUDA_VISIBLE_DEVICES=$cuda_list ctest -I $i,,$NUM_PROC -R "($testcases)" -V &
fi
else
if [[ $cardnumber == $CUDA_DEVICE_COUNT ]]; then
ctest -I $i,,$NUM_PROC -R "($testcases)" --output-on-failure &
else
# echo "env CUDA_VISIBLE_DEVICES=$cuda_list ctest -I $i,,$NUM_PROC -R \"($testcases)\" --output-on-failure &"
env CUDA_VISIBLE_DEVICES=$cuda_list ctest -I $i,,$NUM_PROC -R "($testcases)" --output-on-failure &
fi
fi
done
wait; # wait for all subshells to finish
}
function aggresive_test() {
mkdir -p ${PADDLE_ROOT}/build
cd ${PADDLE_ROOT}/build
if [ ${WITH_TESTING:-ON} == "ON" ] ; then
cat <<EOF
========================================
Running unit tests ...
========================================
EOF
set +x
EXIT_CODE=0;
test_cases=$(ctest -N -V)
exclusive_tests=''
single_card_tests=''
multiple_card_tests=''
is_exclusive=''
is_multicard=''
while read -r line; do
if [[ "$line" == "" ]]; then
continue
fi
read matchstr <<< $(echo "$line"|grep -oEi 'Test[ \t]+#')
if [[ "$matchstr" == "" ]]; then
# Any test case with LABELS property would be parse here
# RUN_TYPE=EXCLUSIVE mean the case would run exclusively
# RUN_TYPE=DIST mean the case would take two graph cards during runtime
read is_exclusive <<< $(echo "$line"|grep -oEi "RUN_TYPE=EXCLUSIVE")
read is_multicard <<< $(echo "$line"|grep -oEi "RUN_TYPE=DIST")
continue
fi
read testcase <<< $(echo "$line"|grep -oEi "\w+$")
if [[ "$is_multicard" == "" ]]; then
# trick: treat all test case with prefix "test_dist" as dist case, and would run on 2 cards
read is_multicard <<< $(echo "$testcase"|grep -oEi "test_dist")
fi
if [[ "$is_exclusive" != "" ]]; then
if [[ "$exclusive_tests" == "" ]]; then
exclusive_tests="^$testcase$"
else
exclusive_tests="$exclusive_tests|^$testcase$"
fi
elif [[ "$is_multicard" != "" ]]; then
if [[ "$multiple_card_tests" == "" ]]; then
multiple_card_tests="^$testcase$"
else
multiple_card_tests="$multiple_card_tests|^$testcase$"
fi
else
if [[ "$single_card_tests" == "" ]]; then
single_card_tests="^$testcase$"
else
single_card_tests="$single_card_tests|^$testcase$"
fi
fi
is_exclusive=''
is_multicard=''
matchstr=''
testcase=''
done <<< "$test_cases";
card_test "$single_card_tests" 1
card_test "$multiple_card_tests" 2
card_test "$exclusive_tests"
if [[ "$EXIT_CODE" != "0" ]]; then
exit 1;
fi
set -ex
fi
}
function gen_doc_lib() {
mkdir -p ${PADDLE_ROOT}/build
cd ${PADDLE_ROOT}/build
@ -805,7 +994,7 @@ function main() {
gen_dockerfile ${PYTHON_ABI:-""}
;;
test)
run_test
aggresive_test
;;
single_test)
single_test $2
@ -835,7 +1024,7 @@ function main() {
cmake_gen ${PYTHON_ABI:-""}
build ${parallel_number}
assert_api_not_changed ${PYTHON_ABI:-""}
run_test
aggresive_test
gen_fluid_lib ${parallel_number}
test_fluid_lib
assert_api_spec_approvals
@ -868,7 +1057,7 @@ function main() {
cicheck_py35)
cmake_gen ${PYTHON_ABI:-""}
build ${parallel_number}
run_test
aggresive_test
assert_api_not_changed ${PYTHON_ABI:-""}
;;
cmake_gen)
@ -882,7 +1071,7 @@ function main() {
;;
*)
print_usage
exit 0
exit 1
;;
esac
}

1
python/paddle/fluid/tests/book/high-level-api/CMakeLists.txt
Unescape View File

@ -23,6 +23,7 @@ else()
message(WARNING "These tests has been disabled in OSX for random fail: \n" ${src})
elseif()
py_test(${src} SRCS ${src}.py)
set_tests_properties(${src} PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE")
endif()
endforeach()
endif()

10
python/paddle/fluid/tests/unittests/CMakeLists.txt
Unescape View File

@ -137,3 +137,13 @@ endif()
if (WITH_MKLDNN)
add_subdirectory(mkldnn)
endif()
if(WITH_DISTRIBUTE)
set_tests_properties(test_listen_and_serv_op test_nce_remote_table_op test_hsigmoid_remote_table_op
PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE")
endif()
set_tests_properties(test_weight_decay test_conv_shift_op test_alloc_continuous_space_op test_recordio_reader
test_parallel_executor_test_while_train test_adam_op_multi_thread test_parallel_executor_mnist
test_parallel_executor_seresnext test_parallel_executor_crf test_nearest_interp_op
PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE")

Write Preview
Loading…
Cancel
Save