Add more log

7 years ago · 3c9cea597e
parent f8f1a963d9
commit 3c9cea597e
1 changed files with 3 additions and 1 deletions
--- a/paddle/fluid/framework/parallel_executor.cc
+++ b/paddle/fluid/framework/parallel_executor.cc
@ -365,6 +365,7 @@ struct NCCLAllReduceOpHandle : public OpHandle {
        auto &p = static_cast<VarHandle *>(in)->place_;
        in->generated_op_->Wait(dev_ctx_[p]);
      }
+      VLOG(3) << "Before NCCL";
      PADDLE_ENFORCE(cudaDeviceSynchronize());

      auto &var_name = static_cast<VarHandle *>(this->inputs_[0])->name_;
@ -394,8 +395,9 @@ struct NCCLAllReduceOpHandle : public OpHandle {
            nccl_ctx.comm, nccl_ctx.stream());
      }
      platform::dynload::ncclGroupEnd();
-
      PADDLE_ENFORCE(cudaDeviceSynchronize());
+
+      VLOG(3) << "After NCCL";
    }
  }
 };