|
|
|
|
@ -40,11 +40,124 @@ AllReduceOpHandle::AllReduceOpHandle(ir::Node *node,
|
|
|
|
|
AllReduceOpHandle::AllReduceOpHandle(ir::Node *node,
|
|
|
|
|
const std::vector<Scope *> &local_scopes,
|
|
|
|
|
const std::vector<platform::Place> &places)
|
|
|
|
|
: OpHandleBase(node), local_scopes_(local_scopes), places_(places) {}
|
|
|
|
|
: OpHandleBase(node), local_scopes_(local_scopes), places_(places) {
|
|
|
|
|
PADDLE_ENFORCE_EQ(places_.size(), local_scopes_.size());
|
|
|
|
|
}
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
void AllReduceOpHandle::RunImpl() {
|
|
|
|
|
platform::RecordEvent record_event(Name());
|
|
|
|
|
|
|
|
|
|
WaitInputVarGenerated();
|
|
|
|
|
std::vector<VarHandleBase *> inputs = this->Inputs();
|
|
|
|
|
std::vector<VarHandleBase *> outputs = this->Outputs();
|
|
|
|
|
auto in_var_handles = DynamicCast<VarHandle>(inputs);
|
|
|
|
|
auto out_var_handles = DynamicCast<VarHandle>(outputs);
|
|
|
|
|
AllReduceImpl(in_var_handles, out_var_handles);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void AllReduceOpHandle::AllReduceImpl(
|
|
|
|
|
const std::vector<VarHandle *> &in_var_handles,
|
|
|
|
|
const std::vector<VarHandle *> &out_var_handles) {
|
|
|
|
|
size_t num_places = places_.size();
|
|
|
|
|
PADDLE_ENFORCE_EQ(
|
|
|
|
|
in_var_handles.size(), num_places,
|
|
|
|
|
"The NoDummyInputSize should be equal to the number of places.");
|
|
|
|
|
PADDLE_ENFORCE_EQ(
|
|
|
|
|
in_var_handles.size(), out_var_handles.size(),
|
|
|
|
|
"The NoDummyInputSize and NoDummyOutputSize should be equal.");
|
|
|
|
|
PADDLE_ENFORCE_EQ(local_exec_scopes_.size(), num_places);
|
|
|
|
|
|
|
|
|
|
std::vector<const void *> lod_tensor_data;
|
|
|
|
|
std::vector<platform::Place> places;
|
|
|
|
|
lod_tensor_data.reserve(num_places);
|
|
|
|
|
places.reserve(num_places);
|
|
|
|
|
int64_t numel = -1;
|
|
|
|
|
bool is_gpu_place = false;
|
|
|
|
|
auto dtype = static_cast<framework::proto::VarType::Type>(0);
|
|
|
|
|
for (size_t i = 0; i < local_exec_scopes_.size(); ++i) {
|
|
|
|
|
auto &local_scope = local_exec_scopes_[i];
|
|
|
|
|
auto var = local_scope->FindVar(in_var_handles[i]->name());
|
|
|
|
|
PADDLE_ENFORCE_NOT_NULL(var, "%s is not found int scope.",
|
|
|
|
|
in_var_handles[i]->name());
|
|
|
|
|
auto &lod_tensor = var->Get<LoDTensor>();
|
|
|
|
|
|
|
|
|
|
if (i == 0) {
|
|
|
|
|
numel = static_cast<int64_t>(lod_tensor.numel());
|
|
|
|
|
dtype = lod_tensor.type();
|
|
|
|
|
is_gpu_place = platform::is_gpu_place(lod_tensor.place());
|
|
|
|
|
}
|
|
|
|
|
PADDLE_ENFORCE_EQ(numel, static_cast<int64_t>(lod_tensor.numel()));
|
|
|
|
|
PADDLE_ENFORCE_EQ(dtype, lod_tensor.type());
|
|
|
|
|
PADDLE_ENFORCE_EQ(is_gpu_place, platform::is_gpu_place(lod_tensor.place()));
|
|
|
|
|
|
|
|
|
|
lod_tensor_data.emplace_back(lod_tensor.data<void>());
|
|
|
|
|
places.emplace_back(lod_tensor.place());
|
|
|
|
|
|
|
|
|
|
VLOG(10) << "place:" << i << ", input_name:" << in_var_handles[i]->name()
|
|
|
|
|
<< ", out_name:" << out_var_handles[i]->name();
|
|
|
|
|
|
|
|
|
|
PADDLE_ENFORCE_EQ(in_var_handles[i]->name(), out_var_handles[i]->name(),
|
|
|
|
|
"The name of input and output should be equal.");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
std::vector<std::string> grad_var_names;
|
|
|
|
|
grad_var_names.reserve(num_places);
|
|
|
|
|
for (auto &out_var : out_var_handles) {
|
|
|
|
|
grad_var_names.emplace_back(out_var->Name());
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
AllReduceFunc(lod_tensor_data, dtype, numel, places, grad_var_names);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void AllReduceOpHandle::AllReduceFunc(
|
|
|
|
|
std::vector<const void *> lod_tensor_data,
|
|
|
|
|
const framework::proto::VarType::Type &dtype, int64_t numel,
|
|
|
|
|
const std::vector<platform::Place> &places,
|
|
|
|
|
const std::vector<std::string> &out_var_names) {
|
|
|
|
|
if (is_gpu_place(places[0])) {
|
|
|
|
|
#if defined(PADDLE_WITH_CUDA) && !defined(_WIN32)
|
|
|
|
|
void AllReduceOpHandle::RunAllReduceFuncs(
|
|
|
|
|
PADDLE_ENFORCE_NOT_NULL(nccl_ctxs_, "nccl_ctxs should not be nullptr.");
|
|
|
|
|
ncclDataType_t nccl_dtype = platform::ToNCCLDataType(dtype);
|
|
|
|
|
std::vector<std::function<void()>> all_reduce_calls;
|
|
|
|
|
for (size_t i = 0; i < local_exec_scopes_.size(); ++i) {
|
|
|
|
|
auto &p = places[i];
|
|
|
|
|
void *buffer = const_cast<void *>(lod_tensor_data.at(i));
|
|
|
|
|
all_reduce_calls.emplace_back([=] {
|
|
|
|
|
NCCLAllReduce(p, buffer, buffer, numel, nccl_dtype, ncclSum);
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
NCCLAllReduceFunc(all_reduce_calls);
|
|
|
|
|
#else
|
|
|
|
|
PADDLE_THROW("Not compiled with CUDA.");
|
|
|
|
|
#endif
|
|
|
|
|
} else { // Special handle CPU only Operator's gradient. Like CRF
|
|
|
|
|
auto &trg = *local_exec_scopes_[0]
|
|
|
|
|
->FindVar(out_var_names[0])
|
|
|
|
|
->GetMutable<LoDTensor>();
|
|
|
|
|
|
|
|
|
|
// Reduce All Tensor to trg in CPU
|
|
|
|
|
ReduceBufferData func(lod_tensor_data, trg.data<void>(), numel);
|
|
|
|
|
VisitDataType(trg.type(), func);
|
|
|
|
|
|
|
|
|
|
for (size_t i = 1; i < local_exec_scopes_.size(); ++i) {
|
|
|
|
|
auto &scope = local_exec_scopes_[i];
|
|
|
|
|
auto &p = places[i];
|
|
|
|
|
auto *var = scope->FindVar(out_var_names[i]);
|
|
|
|
|
|
|
|
|
|
size_t size = numel * SizeOfType(trg.type());
|
|
|
|
|
RunAndRecordEvent(p, [&trg, var, p, size] {
|
|
|
|
|
auto dst_ptr = var->GetMutable<framework::LoDTensor>()->data<void>();
|
|
|
|
|
platform::CPUPlace cpu_place;
|
|
|
|
|
memory::Copy(cpu_place, dst_ptr, cpu_place, trg.data<void>(), size);
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
VLOG(10) << Name() << " size:" << numel * SizeOfType(dtype);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#if defined(PADDLE_WITH_CUDA) && !defined(_WIN32)
|
|
|
|
|
void AllReduceOpHandle::NCCLAllReduceFunc(
|
|
|
|
|
const std::vector<std::function<void()>> &all_reduce_calls) {
|
|
|
|
|
this->RunAndRecordEvent([&] {
|
|
|
|
|
if (all_reduce_calls.size() == 1UL) {
|
|
|
|
|
@ -80,85 +193,6 @@ void AllReduceOpHandle::RunAllReduceFuncs(
|
|
|
|
|
}
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
void AllReduceOpHandle::RunImpl() {
|
|
|
|
|
platform::RecordEvent record_event(Name());
|
|
|
|
|
|
|
|
|
|
WaitInputVarGenerated();
|
|
|
|
|
|
|
|
|
|
auto in_var_handles = DynamicCast<VarHandle>(this->Inputs());
|
|
|
|
|
auto out_var_handles = DynamicCast<VarHandle>(this->Outputs());
|
|
|
|
|
PADDLE_ENFORCE_EQ(
|
|
|
|
|
in_var_handles.size(), places_.size(),
|
|
|
|
|
"The NoDummyInputSize should be equal to the number of places.");
|
|
|
|
|
PADDLE_ENFORCE_EQ(
|
|
|
|
|
in_var_handles.size(), out_var_handles.size(),
|
|
|
|
|
"The NoDummyInputSize and NoDummyOutputSize should be equal.");
|
|
|
|
|
|
|
|
|
|
std::vector<const LoDTensor *> lod_tensors;
|
|
|
|
|
for (size_t i = 0; i < local_scopes_.size(); ++i) {
|
|
|
|
|
auto &local_scope = local_exec_scopes_[i];
|
|
|
|
|
auto &lod_tensor =
|
|
|
|
|
local_scope->FindVar(in_var_handles[i]->name())->Get<LoDTensor>();
|
|
|
|
|
lod_tensors.emplace_back(&lod_tensor);
|
|
|
|
|
VLOG(10) << "place:" << i << ", input_name:" << in_var_handles[i]->name()
|
|
|
|
|
<< ", out_name:" << out_var_handles[i]->name();
|
|
|
|
|
PADDLE_ENFORCE_EQ(in_var_handles[i]->name(), out_var_handles[i]->name(),
|
|
|
|
|
"The name of input and output should be equal.");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (platform::is_gpu_place(lod_tensors[0]->place())) {
|
|
|
|
|
#if defined(PADDLE_WITH_CUDA) && !defined(_WIN32)
|
|
|
|
|
PADDLE_ENFORCE(nccl_ctxs_, "nccl_ctxs should not be nullptr.");
|
|
|
|
|
int dtype = -1;
|
|
|
|
|
size_t numel = 0;
|
|
|
|
|
std::vector<std::function<void()>> all_reduce_calls;
|
|
|
|
|
for (size_t i = 0; i < local_scopes_.size(); ++i) {
|
|
|
|
|
auto &p = places_[i];
|
|
|
|
|
auto &lod_tensor = *lod_tensors[i];
|
|
|
|
|
void *buffer = const_cast<void *>(lod_tensor.data<void>());
|
|
|
|
|
|
|
|
|
|
if (dtype == -1) {
|
|
|
|
|
dtype = platform::ToNCCLDataType(lod_tensor.type());
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (numel == 0) {
|
|
|
|
|
numel = static_cast<size_t>(lod_tensor.numel());
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
all_reduce_calls.emplace_back([=] {
|
|
|
|
|
NCCLAllReduce(p, buffer, buffer, numel,
|
|
|
|
|
static_cast<ncclDataType_t>(dtype), ncclSum);
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
VLOG(10) << "allreduce size:" << numel * SizeOfType(lod_tensors[0]->type());
|
|
|
|
|
RunAllReduceFuncs(all_reduce_calls);
|
|
|
|
|
#else
|
|
|
|
|
PADDLE_THROW("Not compiled with CUDA");
|
|
|
|
|
#endif
|
|
|
|
|
} else { // Special handle CPU only Operator's gradient. Like CRF
|
|
|
|
|
auto &trg = *this->local_exec_scopes_[0]
|
|
|
|
|
->FindVar(out_var_handles[0]->name())
|
|
|
|
|
->GetMutable<framework::LoDTensor>();
|
|
|
|
|
|
|
|
|
|
// Reduce All Tensor to trg in CPU
|
|
|
|
|
ReduceLoDTensor func(lod_tensors, &trg);
|
|
|
|
|
VisitDataType(lod_tensors[0]->type(), func);
|
|
|
|
|
|
|
|
|
|
for (size_t i = 1; i < local_scopes_.size(); ++i) {
|
|
|
|
|
auto &scope = local_exec_scopes_[i];
|
|
|
|
|
auto &p = places_[i];
|
|
|
|
|
auto *var = scope->FindVar(out_var_handles[i]->name());
|
|
|
|
|
auto *dev_ctx = dev_ctxes_.at(p);
|
|
|
|
|
|
|
|
|
|
RunAndRecordEvent(p, [&trg, var, dev_ctx, p] {
|
|
|
|
|
auto &tensor_gpu = *var->GetMutable<framework::LoDTensor>();
|
|
|
|
|
auto &tensor_cpu = trg;
|
|
|
|
|
TensorCopy(tensor_cpu, p, *dev_ctx, &tensor_gpu);
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
std::string AllReduceOpHandle::Name() const { return "all_reduce"; }
|
|
|
|
|
} // namespace details
|
|
|
|
|
} // namespace framework
|
|
|
|
|
|
chengduo
5 years ago
committed by
GitHub