diff --git a/paddle/fluid/framework/parallel_executor.cc b/paddle/fluid/framework/parallel_executor.cc
index 25f8d7afde..66ad3f33d9 100644
--- a/paddle/fluid/framework/parallel_executor.cc
+++ b/paddle/fluid/framework/parallel_executor.cc
@@ -690,6 +690,10 @@ void ParallelExecutor::BCastParamsToGPUs(
       }
       platform::dynload::ncclGroupEnd();
     }
+
+    for (auto &stream : member_->communication_streams_) {
+      stream.second.ctx_->Wait();
+    }
   }
 #else
   PADDLE_THROW("Not compiled with CUDA");