m53297601
/
Paddle
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Decrease threshold for failed ut retry (#30903)

Browse Source 
* Decrease threshold for failed ut retry

* retry Method upgrade

* second method upgrade

* fix error

* Remove the comment lines

* test for modified_retry_times

* fix error

* fix some error

* fix error

* fix error

* remove test content

* fix error

* Reduce duplicate code

* fix more than 10 ut failed bug

* fix more than 10 ut failed  bug on mac
test_model_benchmark_ci
YUNSHEN XIE 4 years ago committed by GitHub
parent 2e9e3fad15
commit 2111d912d4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 46 additions and 45 deletions
Show Stats Download Patch File Download Diff File

91
paddle/scripts/paddle_build.sh
Unescape View File

@ -615,18 +615,19 @@ EOF
retry_time=3
exec_times=0
exec_time_array=('first' 'second' 'third')
exec_retry_threshold=20
exec_retry_threshold=10
is_retry_execuate=0
if [ -n "$failed_test_lists" ];then
mactest_error=1
read need_retry_ut_str <<< $(echo "$failed_test_lists" | grep -oEi "\-.+\(" | sed 's/(//' | sed 's/- //' )
need_retry_ut_arr=(${need_retry_ut_str})
need_retry_ut_count=${#need_retry_ut_arr[@]}
read retry_unittests <<< $(echo "$failed_test_lists" | grep -oEi "\-.+\(" | sed 's/(//' | sed 's/- //' )
if [ $need_retry_ut_count -lt $exec_retry_threshold ];then
while ( [ $exec_times -lt $retry_time ] && [ -n "${failed_test_lists}" ] )
while ( [ $exec_times -lt $retry_time ] )
do
retry_unittests_record="$retry_unittests_record$failed_test_lists"
failed_test_lists_ult=`echo "${failed_test_lists}"`
read retry_unittests <<< $(echo "$failed_test_lists" | grep -oEi "\-.+\(" | sed 's/(//' | sed 's/- //' )
echo "========================================="
echo "This is the ${exec_time_array[$exec_times]} time to re-run"
echo "========================================="
@ -650,9 +651,8 @@ EOF
exec_times=$[$exec_times+1]
done
else
echo "========================================="
echo "There are more than 20 failed unit tests, so no unit test retry!!!"
echo "========================================="
# There are more than 10 failed unit tests, so no unit test retry
is_retry_execuate=1
fi
fi
@ -665,24 +665,10 @@ EOF
set +x
export http_proxy=$my_proxy
export https_proxy=$my_proxy
set -x
if [ "$mactest_error" != 0 ];then
if [[ "$failed_test_lists" == "" ]]; then
echo "========================================"
echo "There are failed tests, which have been successful after re-run:"
echo "========================================"
echo "The following tests have been re-ran:"
echo "${retry_unittests_record}"
else
failed_test_lists_ult=`echo "${failed_test_lists}"`
echo "========================================"
echo "Summary Failed Tests... "
echo "========================================"
echo "The following tests FAILED: "
echo "${failed_test_lists_ult}"
exit 8;
fi
show_ut_retry_result
fi
set -x
fi
}
@ -1204,18 +1190,18 @@ set +x
retry_unittests_record=''
retry_time=3
exec_time_array=('first' 'second' 'third')
exec_retry_threshold=20
exec_retry_threshold=10
is_retry_execuate=0
if [ -n "$failed_test_lists" ];then
read need_retry_ut_str <<< $(echo "$failed_test_lists" | grep -oEi "\-.+\(.+\)" | sed 's/(.\+)//' | sed 's/- //' )
need_retry_ut_arr=(${need_retry_ut_str})
need_retry_ut_count=${#need_retry_ut_arr[@]}
read retry_unittests <<< $(echo "$failed_test_lists" | grep -oEi "\-.+\(.+\)" | sed 's/(.\+)//' | sed 's/- //' )
if [ $need_retry_ut_count -lt $exec_retry_threshold ];then
while ( [ $exec_times -lt $retry_time ] && [ -n "${failed_test_lists}" ] )
while ( [ $exec_times -lt $retry_time ] )
do
retry_unittests_record="$retry_unittests_record$failed_test_lists"
failed_test_lists_ult=`echo "${failed_test_lists}" |grep -Po '[^ ].*$'`
read retry_unittests <<< $(echo "$failed_test_lists" | grep -oEi "\-.+\(.+\)" | sed 's/(.\+)//' | sed 's/- //' )
echo "========================================="
echo "This is the ${exec_time_array[$exec_times]} time to re-run"
echo "========================================="
@ -1270,36 +1256,51 @@ set +x
one_card_retry=''
multiple_card_retry=''
exclusive_retry=''
retry_unittests=''
done
else
echo "========================================="
echo "There are more than 20 failed unit tests, so no unit test retry!!!"
echo "========================================="
# There are more than 10 failed unit tests, so no unit test retry
is_retry_execuate=1
fi
fi
if [[ "$EXIT_CODE" != "0" ]]; then
if [[ "$failed_test_lists" == "" ]]; then
echo "========================================"
echo "There are failed tests, which have been successful after re-run:"
echo "========================================"
echo "The following tests have been re-ran:"
echo "${retry_unittests_record}"
else
failed_test_lists_ult=`echo "${failed_test_lists}" |grep -Po '[^ ].*$'`
echo "========================================"
echo "Summary Failed Tests... "
echo "========================================"
echo "The following tests FAILED: "
echo "${failed_test_lists_ult}"
exit 8;
fi
show_ut_retry_result
fi
set -ex
fi
}
function show_ut_retry_result() {
if [[ "$is_retry_execuate" != "0" ]];then
failed_test_lists_ult=`echo "${failed_test_lists}" | grep -Po '[^ ].*$'`
echo "========================================="
echo "There are more than 10 failed unit tests, so no unit test retry!!!"
echo "========================================="
echo "The following tests FAILED: "
echo "${failed_test_lists_ult}"
exit 8;
else
read retry_unittests_ut_name <<< $(echo "$retry_unittests_record" | grep -oEi "\-.+\(" | sed 's/(//' | sed 's/- //' )
retry_unittests_record_judge=$(echo ${retry_unittests_ut_name}| tr ' ' '\n' | sort | uniq -c | awk '{if ($1 >=3) {print $2}}')
if [ -z "${retry_unittests_record_judge}" ];then
echo "========================================"
echo "There are failed tests, which have been successful after re-run:"
echo "========================================"
echo "The following tests have been re-ran:"
echo "${retry_unittests_record}"
else
failed_ut_re=$(echo "${retry_unittests_record_judge}" | awk BEGIN{RS=EOF}'{gsub(/\n/,"|");print}')
echo "========================================"
echo "There are failed tests, which have been executed re-run,but success rate is less than 50%:"
echo "Summary Failed Tests... "
echo "========================================"
echo "The following tests FAILED: "
echo "${retry_unittests_record}" | grep -E "$failed_ut_re"
exit 8;
fi
fi
}
function parallel_test_base_cpu() {
mkdir -p ${PADDLE_ROOT}/build
cd ${PADDLE_ROOT}/build

Write Preview
Loading…
Cancel
Save