Add debug code

7 years ago · f8f1a963d9
parent fbbcedda01
commit f8f1a963d9
1 changed files with 3 additions and 0 deletions
--- a/paddle/fluid/framework/parallel_executor.cc
+++ b/paddle/fluid/framework/parallel_executor.cc
@ -365,6 +365,7 @@ struct NCCLAllReduceOpHandle : public OpHandle {
        auto &p = static_cast<VarHandle *>(in)->place_;
        in->generated_op_->Wait(dev_ctx_[p]);
      }
+      PADDLE_ENFORCE(cudaDeviceSynchronize());

      auto &var_name = static_cast<VarHandle *>(this->inputs_[0])->name_;
      int dtype = -1;
@ -393,6 +394,8 @@ struct NCCLAllReduceOpHandle : public OpHandle {
            nccl_ctx.comm, nccl_ctx.stream());
      }
      platform::dynload::ncclGroupEnd();
+
+      PADDLE_ENFORCE(cudaDeviceSynchronize());
    }
  }
 };