m53297601
/
Paddle
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add the ipipe log param prefix (#29545)

Browse Source 
* Add the ipipe log param prefix

1. add the prefix;
2. using Colon before the metric values;

* 增加效率云日志指标收集前缀

暂未验证windows bat的这个字符串替换是否正常

* Preserve The Old Format Metrics During The Transition Period

Please DELETE the old format metrics log finally.
The period man last for a week.

* ipipe_log_param + ccache and clcache ..
revert-31562-mean
Ren Wei (任卫) 4 years ago committed by GitHub
parent 2a01756bf3
commit 95df0e1447
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 25 additions and 1 deletions
Show Stats Download Patch File Download Diff File

9
paddle/scripts/paddle_build.bat
Unescape View File

@ -98,9 +98,11 @@ git diff --name-only %BRANCH% | findstr /V "\.py" || set CI_SKIP_CPP_TEST=ON
:mkbuild
if not exist build (
echo Windows build cache FALSE
echo "ipipe_log_param_Windows_Build_Cache: FALSE"
mkdir build
) else (
echo Windows build cache TRUE
echo "ipipe_log_param_Windows_Build_Cache: TRUE"
)
cd /d build
dir .
@ -329,10 +331,12 @@ set /p libsize=< lib_size.txt
for /F %%i in ("%libsize%") do (
set /a libsize_m=%%i/1024
echo "Windows Paddle_Inference Size: !libsize_m!M"
echo "ipipe_log_param_Windows_Paddle_Inference_Size: !libsize_m!M"
)
%cache_dir%\tools\busybox64.exe du -h -d 0 %cd%\python\dist > whl_size.txt
set /p whlsize=< whl_size.txt
for /F %%i in ("%whlsize%") do echo "Windows PR whl Size: %%i"
for /F %%i in ("%whlsize%") do echo "ipipe_log_param_Windows_PR_whl_Size: %%i"
dir /s /b python\dist\*.whl > whl_file.txt
set /p PADDLE_WHL_FILE_WIN=< whl_file.txt
@ -485,6 +489,7 @@ echo spec_path=$(pwd)/UNITTEST_PR.spec>> check_change_of_unittest.sh
echo ctest -N ^| awk -F ':' '{print $2}' ^| sed '/^^$/d' ^| sed '$d' ^> ${spec_path}>> check_change_of_unittest.sh
echo num=$(awk 'END{print NR}' ${spec_path})>> check_change_of_unittest.sh
echo echo "Windows 1 card TestCases count is $num">> check_change_of_unittest.sh
echo echo "ipipe_log_param_Windows_1_Card_TestCases_Count: $num">> check_change_of_unittest.sh
echo UPSTREAM_URL='https://github.com/PaddlePaddle/Paddle'>> check_change_of_unittest.sh
echo origin_upstream_url=`git remote -v ^| awk '{print $1, $2}' ^| uniq ^| grep upstream ^| awk '{print $2}'`>> check_change_of_unittest.sh
echo if [ "$origin_upstream_url" == "" ]; then>> check_change_of_unittest.sh
@ -575,6 +580,8 @@ set /a ss=100%ss%%%100
set /a end_secs=dd*86400+hh*3600+nn*60+ss
set /a cost_secs=end_secs-start_sec
echo "Windows %~3 Time: %cost_secs%s"
set tempTaskName=%~3
echo "ipipe_log_param_Windows_%tempTaskName: =_%_Time: %cost_secs%s"
goto:eof
@ -582,9 +589,11 @@ goto:eof
for /f "tokens=2,4" %%i in ('clcache.exe -s ^| findstr "entries hits"') do set %%i=%%j
if %hits% EQU 0 (
echo "clcache hit rate: 0%%"
echo "ipipe_log_param_Clcache_Hit_Rate: 0%%"
) else (
set /a rate=%hits%*10000/%entries%
echo "clcache hit rate: %rate:~0,-2%.%rate:~-2%%%"
echo "ipipe_log_param_Clcache_Hit_Hate: %rate:~0,-2%.%rate:~-2%%%"
)
goto:eof

17
paddle/scripts/paddle_build.sh
Unescape View File

@ -391,6 +391,7 @@ EOF
tar -czf paddle_inference.tgz paddle_inference
buildSize=$(du -h --max-depth=0 ${PADDLE_ROOT}/build/paddle_inference.tgz |awk '{print $1}')
echo "Paddle_Inference Size: $buildSize"
echo "ipipe_log_param_Paddle_Inference_Size: $buildSize"
else
SYSTEM=`uname -s`
if [ "$SYSTEM" == "Darwin" ]; then
@ -400,8 +401,10 @@ EOF
fi
buildSize=$($com ${PADDLE_ROOT}/build |awk '{print $1}')
echo "Build Size: $buildSize"
echo "ipipe_log_param_Build_Size: $buildSize"
PR_whlSize=$($com ${PADDLE_ROOT}/build/python/dist |awk '{print $1}')
echo "PR whl Size: $PR_whlSize"
echo "ipipe_log_param_PR_whl_Size: $PR_whlSize"
fi
}
@ -426,6 +429,7 @@ function cmake_gen_and_build() {
build $2
endTime_s=`date +%s`
echo "Build Time: $[ $endTime_s - $startTime_s ]s"
echo "ipipe_log_param_Build_Time: $[ $endTime_s - $startTime_s ]s"
}
function build_mac() {
@ -463,6 +467,7 @@ function cmake_gen_and_build_mac() {
build_mac
endTime_s=`date +%s`
echo "Build Time: $[ $endTime_s - $startTime_s ]s"
echo "ipipe_log_param_Build_Time: $[ $endTime_s - $startTime_s ]s"
}
function run_test() {
@ -650,6 +655,7 @@ EOF
#mactest_error=$?
ut_endTime_s=`date +%s`
echo "Mac testCase Time: $[ $ut_endTime_s - $ut_startTime_s ]s"
echo "ipipe_log_param_Mac_TestCases_Time: $[ $ut_endTime_s - $ut_startTime_s ]s"
paddle version
# Recovery proxy to avoid failure in later steps
set +x
@ -932,8 +938,10 @@ EOF
num=$(echo $testcases|grep -o '\^'|wc -l)
if [ "$2" == "" ]; then
echo "exclusive TestCases count is $num"
echo "ipipe_log_param_Exclusive_TestCases_Count: $num"
else
echo "$2 card TestCases count is $num"
echo "ipipe_log_param_${2}_Cards_TestCases_Count $num"
fi
}
@ -1025,8 +1033,10 @@ function card_test() {
ut_endTime_s=`date +%s`
if [ "$2" == "" ]; then
echo "exclusive TestCases Total Time: $[ $ut_endTime_s - $ut_startTime_s ]s"
echo "ipipe_log_param_Exclusive_TestCases_Total_Time: $[ $ut_endTime_s - $ut_startTime_s ]s"
else
echo "$2 card TestCases Total Time: $[ $ut_endTime_s - $ut_startTime_s ]s"
echo "ipipe_log_param_${2}_Cards_TestCases_Total_Time: $[ $ut_endTime_s - $ut_startTime_s ]s"
fi
set +m
}
@ -1325,6 +1335,7 @@ function parallel_test() {
fi
ut_total_endTime_s=`date +%s`
echo "TestCases Total Time: $[ $ut_total_endTime_s - $ut_total_startTime_s ]s"
echo "ipipe_log_param_TestCases_Total_Time: $[ $ut_total_endTime_s - $ut_total_startTime_s ]s"
}
function enable_unused_var_check() {
@ -1604,6 +1615,7 @@ EOF
fi
endTime_s=`date +%s`
echo "Build Time: $[ $endTime_s - $startTime_s ]s"
echo "ipipe_log_param_Build_Time: $[ $endTime_s - $startTime_s ]s"
build_size "paddle_inference"
}
@ -1634,7 +1646,8 @@ EOF
${TENSORRT_LIB_DIR:-/usr/local/TensorRT/lib}
EXIT_CODE=$?
fluid_endTime_s=`date +%s`
echo "test_fluid_lib Total Time: $[ $fluid_endTime_s - $fluid_startTime_s ]s"
echo "test_fluid_lib Total Time: $[ $fluid_endTime_s - $fluid_startTime_s ]s"
echo "ipipe_log_param_Test_Fluid_Lib_Total_Time: $[ $fluid_endTime_s - $fluid_startTime_s ]s"
./clean.sh
if [[ "$EXIT_CODE" != "0" ]]; then
exit 8;
@ -1653,6 +1666,7 @@ EOF
EXIT_CODE=$?
fluid_train_endTime_s=`date +%s`
echo "test_fluid_lib_train Total Time: $[ $fluid_train_endTime_s - $fluid_train_startTime_s ]s"
echo "ipipe_log_param_Test_Fluid_Lib_Train_Total_Time: $[ $fluid_train_endTime_s - $fluid_train_startTime_s ]s"
./clean.sh
if [[ "$EXIT_CODE" != "0" ]]; then
exit 8;
@ -1680,6 +1694,7 @@ function example() {
function collect_ccache_hits() {
rate=$(ccache -s | grep 'cache hit rate' | awk '{print $4}')
echo "ccache hit rate: ${rate}%"
echo "ipipe_log_param_Ccache_Hit_Rate: ${rate}%"
}

Write Preview
Loading…
Cancel
Save