From 9aa05b177f974eca72d36855e5aeaebe2fbd59ac Mon Sep 17 00:00:00 2001
From: chendongsheng <chendongsheng1@huawei.com>
Date: Sun, 21 Mar 2021 14:40:00 +0800
Subject: [PATCH] fixed the wait usage of shell script

---
 .../ps/cmp_sparse_embedding/shell_run_test.sh  | 16 ++++++++++++++--
 tests/st/ps/full_ps/shell_run_test.sh          | 16 ++++++++++++++--
 tests/st/ps/multi_full_ps/shell_run_test.sh    | 18 ++++++++++++++++--
 tests/st/ps/part_ps/shell_run_test.sh          | 17 +++++++++++++++--
 4 files changed, 59 insertions(+), 8 deletions(-)

diff --git a/tests/st/ps/cmp_sparse_embedding/shell_run_test.sh b/tests/st/ps/cmp_sparse_embedding/shell_run_test.sh
index 44b294bf1d..41fb141917 100644
--- a/tests/st/ps/cmp_sparse_embedding/shell_run_test.sh
+++ b/tests/st/ps/cmp_sparse_embedding/shell_run_test.sh
@@ -42,13 +42,25 @@ do
 done
 
 export MS_ROLE=MS_WORKER
+process_pid=()
 for((i=0;i<$MS_WORKER_NUM;i++));
 do
   rm -rf ${execute_path}/worker_$i/
   mkdir ${execute_path}/worker_$i/
   cd ${execute_path}/worker_$i/ || exit
   python ${self_path}/../test_cmp_sparse_embedding.py --device_target=$DEVICE_TARGET &
+  process_pid[${i}]=`echo $!`
 done
 
-wait $!
-exit $?
+for((i=0; i<${MS_WORKER_NUM}; i++)); do
+    wait ${process_pid[i]}
+    status=`echo $?`
+    if [ "${status}" != "0" ]; then
+        echo "[ERROR] test_cmp_sparse_embedding failed. status: ${status}"
+        exit 1
+    else
+        echo "[INFO] test_cmp_sparse_embedding success."
+    fi
+done
+
+exit 0
diff --git a/tests/st/ps/full_ps/shell_run_test.sh b/tests/st/ps/full_ps/shell_run_test.sh
index e61c9325fe..460a5dd86b 100644
--- a/tests/st/ps/full_ps/shell_run_test.sh
+++ b/tests/st/ps/full_ps/shell_run_test.sh
@@ -43,13 +43,25 @@ do
 done
 
 export MS_ROLE=MS_WORKER
+process_pid=()
 for((i=0;i<$MS_WORKER_NUM;i++));
 do
   rm -rf ${execute_path}/worker_$i/
   mkdir ${execute_path}/worker_$i/
   cd ${execute_path}/worker_$i/ || exit
   python ${self_path}/../test_full_ps_lenet.py --device_target=$DEVICE_TARGET --dataset_path=$DATASET_PATH &
+  process_pid[${i}]=`echo $!`
 done
 
-wait $!
-exit $?
+for((i=0; i<${MS_WORKER_NUM}; i++)); do
+    wait ${process_pid[i]}
+    status=`echo $?`
+    if [ "${status}" != "0" ]; then
+        echo "[ERROR] test_full_ps_lenet failed. status: ${status}"
+        exit 1
+    else
+        echo "[INFO] test_full_ps_lenet success."
+    fi
+done
+
+exit 0
diff --git a/tests/st/ps/multi_full_ps/shell_run_test.sh b/tests/st/ps/multi_full_ps/shell_run_test.sh
index 0a4adc393c..f4979cfd1f 100644
--- a/tests/st/ps/multi_full_ps/shell_run_test.sh
+++ b/tests/st/ps/multi_full_ps/shell_run_test.sh
@@ -44,6 +44,7 @@ do
 done
 
 export MS_ROLE=MS_WORKER
+process_pid=()
 if [ $DEVICE_TARGET == "Ascend" ];then
 for((i=0;i<$MS_WORKER_NUM;i++));
 do
@@ -53,14 +54,27 @@ do
   export RANK_ID=$i
   export DEVICE_ID=$i
   python ${self_path}/../test_multi_full_ps.py --device_target=$DEVICE_TARGET &
+  process_pid[${i}]=`echo $!`
+done
+
+for((i=0; i<${MS_WORKER_NUM}; i++)); do
+    wait ${process_pid[i]}
+    status=`echo $?`
+    if [ "${status}" != "0" ]; then
+        echo "[ERROR] test_multi_full_ps failed. status: ${status}"
+        exit 1
+    else
+        echo "[INFO] test_multi_full_ps success."
+    fi
 done
 fi
+
 if [ $DEVICE_TARGET == "GPU" ];then
   rm -rf ${execute_path}/worker/
   mkdir ${execute_path}/worker/
   cd ${execute_path}/worker/ || exit
   mpirun -n $MS_WORKER_NUM python ${self_path}/../test_multi_full_ps.py --device_target=$DEVICE_TARGET &
+  wait $!
 fi
 
-wait $!
-exit $?
+exit 0
diff --git a/tests/st/ps/part_ps/shell_run_test.sh b/tests/st/ps/part_ps/shell_run_test.sh
index ca42c5daf7..19c74734d2 100644
--- a/tests/st/ps/part_ps/shell_run_test.sh
+++ b/tests/st/ps/part_ps/shell_run_test.sh
@@ -43,13 +43,26 @@ do
 done
 
 export MS_ROLE=MS_WORKER
+process_pid=()
 for((i=0;i<$MS_WORKER_NUM;i++));
 do
   rm -rf ${execute_path}/worker_$i/
   mkdir ${execute_path}/worker_$i/
   cd ${execute_path}/worker_$i/ || exit
   python ${self_path}/../test_ps_embedding_heterogeneous_conv2d_adam.py --device_target=$DEVICE_TARGET --dataset_path=$DATASET_PATH &
+  process_pid[${i}]=`echo $!`
 done
 
-wait $!
-exit $?
+
+for((i=0; i<${MS_WORKER_NUM}; i++)); do
+    wait ${process_pid[i]}
+    status=`echo $?`
+    if [ "${status}" != "0" ]; then
+        echo "[ERROR] test_ps_embedding_heterogeneous_conv2d_adam failed. status: ${status}"
+        exit 1
+    else
+        echo "[INFO] test_ps_embedding_heterogeneous_conv2d_adam success."
+    fi
+done
+
+exit 0