[Kunlun] add gen_bkcl_id_op, support multi XPU cards training using multiprocess (#30858)
liuyuhui
5 years ago
committed by
GitHub
parent
39f41cb47f
commit
4a8b8b4547
@ -0,0 +1,119 @@
|
||||
/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
#include <string>
|
||||
|
||||
#include "glog/logging.h"
|
||||
#include "paddle/fluid/framework/op_proto_maker.h"
|
||||
#include "paddle/fluid/framework/op_registry.h"
|
||||
#include "paddle/fluid/framework/operator.h"
|
||||
#include "paddle/fluid/framework/scope.h"
|
||||
#include "paddle/fluid/framework/var_type_traits.h"
|
||||
#include "paddle/fluid/platform/device_context.h"
|
||||
#include "paddle/fluid/platform/enforce.h"
|
||||
#include "paddle/fluid/platform/place.h"
|
||||
|
||||
#include "paddle/fluid/platform/gen_comm_id_helper.h"
|
||||
|
||||
namespace paddle {
|
||||
namespace operators {
|
||||
|
||||
static void GenBKCLID(std::vector<BKCLUniqueId>* bkcl_ids) {
|
||||
for (size_t i = 0; i < bkcl_ids->size(); ++i) {
|
||||
BKCLResult_t ret = bkcl_get_unique_id(&(*bkcl_ids)[i]);
|
||||
PADDLE_ENFORCE_EQ(BKCL_SUCCESS, ret,
|
||||
platform::errors::PreconditionNotMet(
|
||||
"bkcl get unique id failed [%d]", ret));
|
||||
}
|
||||
}
|
||||
|
||||
static void CopyBKCLIDToVar(const std::vector<BKCLUniqueId>& bkcl_ids,
|
||||
std::function<std::string(size_t)> func,
|
||||
const framework::Scope& scope) {
|
||||
for (size_t i = 0; i < bkcl_ids.size(); ++i) {
|
||||
std::string var_name = func(i);
|
||||
auto var = scope.FindVar(var_name);
|
||||
PADDLE_ENFORCE_NOT_NULL(
|
||||
var, platform::errors::NotFound("Variable with name %s is not found",
|
||||
var_name.c_str()));
|
||||
auto bkcl_id = var->GetMutable<BKCLUniqueId>();
|
||||
memcpy(bkcl_id, &bkcl_ids[i], sizeof(BKCLUniqueId));
|
||||
}
|
||||
}
|
||||
|
||||
class CGenBKCLIdOp : public framework::OperatorBase {
|
||||
public:
|
||||
CGenBKCLIdOp(const std::string& type,
|
||||
const framework::VariableNameMap& inputs,
|
||||
const framework::VariableNameMap& outputs,
|
||||
const framework::AttributeMap& attrs)
|
||||
: OperatorBase(type, inputs, outputs, attrs) {}
|
||||
|
||||
void RunImpl(const framework::Scope& scope,
|
||||
const platform::Place& dev_place) const override {
|
||||
int rank = Attr<int>("rank");
|
||||
framework::Scope& local_scope = scope.NewScope();
|
||||
|
||||
std::function<std::string(size_t)> func = [&](size_t i) -> std::string {
|
||||
return Output("Out");
|
||||
};
|
||||
|
||||
std::vector<BKCLUniqueId> bkcl_ids;
|
||||
bkcl_ids.resize(1);
|
||||
|
||||
if (rank == 0) {
|
||||
GenBKCLID(&bkcl_ids);
|
||||
std::vector<std::string> endpoint_list =
|
||||
Attr<std::vector<std::string>>("other_endpoints");
|
||||
platform::SendBroadCastCommID(endpoint_list, &bkcl_ids);
|
||||
} else {
|
||||
std::string endpoint = Attr<std::string>("endpoint");
|
||||
platform::RecvBroadCastCommID(endpoint, &bkcl_ids);
|
||||
}
|
||||
|
||||
CopyBKCLIDToVar(bkcl_ids, func, scope);
|
||||
scope.DeleteScope(&local_scope);
|
||||
}
|
||||
};
|
||||
|
||||
class CGenBKCLIdOpMaker : public framework::OpProtoAndCheckerMaker {
|
||||
public:
|
||||
void Make() override {
|
||||
AddOutput("Out", "Raw variable contains a BKCL UniqueId instaces.");
|
||||
AddComment(R"DOC(
|
||||
CGenBKCLId operator
|
||||
|
||||
For trainer 0: generate a new UniqueId and send it to all the other trainers.
|
||||
For trainer 1~n: start a gRPC server to get the UniqueId, once got, stop the server.
|
||||
)DOC");
|
||||
AddAttr<std::string>("endpoint",
|
||||
"(string), e.g. 127.0.0.1:6175 "
|
||||
"current listen endpoint");
|
||||
AddAttr<std::vector<std::string>>(
|
||||
"other_endpoints",
|
||||
"['trainer1_ip:port', 'trainer2_ip:port', ...] "
|
||||
"list of other trainer endpoints")
|
||||
.SetDefault({});
|
||||
AddAttr<int>("rank",
|
||||
"(int default 0) "
|
||||
"The rank of the trainer in distributed training.")
|
||||
.SetDefault(0);
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace operators
|
||||
} // namespace paddle
|
||||
|
||||
namespace ops = paddle::operators;
|
||||
|
||||
REGISTER_OPERATOR(c_gen_bkcl_id, ops::CGenBKCLIdOp, ops::CGenBKCLIdOpMaker);
|
||||
@ -0,0 +1,194 @@
|
||||
/* Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#include <ostream>
|
||||
#include <string>
|
||||
|
||||
#include "glog/logging.h"
|
||||
#include "paddle/fluid/framework/op_proto_maker.h"
|
||||
#include "paddle/fluid/framework/op_registry.h"
|
||||
#include "paddle/fluid/framework/operator.h"
|
||||
#include "paddle/fluid/framework/scope.h"
|
||||
#include "paddle/fluid/framework/var_type_traits.h"
|
||||
#include "paddle/fluid/platform/bkcl_helper.h"
|
||||
#include "paddle/fluid/platform/device_context.h"
|
||||
#include "paddle/fluid/platform/enforce.h"
|
||||
#include "paddle/fluid/platform/place.h"
|
||||
#include "paddle/fluid/string/split.h"
|
||||
|
||||
#include "paddle/fluid/platform/gen_comm_id_helper.h"
|
||||
|
||||
namespace paddle {
|
||||
namespace operators {
|
||||
|
||||
static void GenBKCLID(std::vector<BKCLUniqueId>* bkcl_ids) {
|
||||
for (size_t i = 0; i < bkcl_ids->size(); ++i) {
|
||||
BKCLResult_t ret = bkcl_get_unique_id(&(*bkcl_ids)[i]);
|
||||
PADDLE_ENFORCE_EQ(BKCL_SUCCESS, ret,
|
||||
platform::errors::PreconditionNotMet(
|
||||
"bkcl get unique id failed [%d]", ret));
|
||||
}
|
||||
}
|
||||
|
||||
static void CopyBKCLIDToVar(const std::vector<BKCLUniqueId>& bkcl_ids,
|
||||
std::function<std::string(size_t)> func,
|
||||
const framework::Scope& scope) {
|
||||
for (size_t i = 0; i < bkcl_ids.size(); ++i) {
|
||||
std::string var_name = func(i);
|
||||
auto var = scope.FindVar(var_name);
|
||||
PADDLE_ENFORCE_NOT_NULL(
|
||||
var, platform::errors::NotFound("Variable with name %s is not found",
|
||||
var_name.c_str()));
|
||||
auto bkcl_id = var->GetMutable<BKCLUniqueId>();
|
||||
memcpy(bkcl_id, &bkcl_ids[i], sizeof(BKCLUniqueId));
|
||||
}
|
||||
}
|
||||
|
||||
class GenBKCLIdOp : public framework::OperatorBase {
|
||||
public:
|
||||
GenBKCLIdOp(const std::string& type, const framework::VariableNameMap& inputs,
|
||||
const framework::VariableNameMap& outputs,
|
||||
const framework::AttributeMap& attrs)
|
||||
: OperatorBase(type, inputs, outputs, attrs) {}
|
||||
|
||||
void RunImpl(const framework::Scope& scope,
|
||||
const platform::Place& dev_place) const override {
|
||||
std::vector<std::string> trainers =
|
||||
Attr<std::vector<std::string>>("trainers");
|
||||
int trainer_id = Attr<int>("trainer_id");
|
||||
std::string endpoint = trainers[trainer_id];
|
||||
|
||||
PADDLE_ENFORCE_GE(trainer_id, 0, platform::errors::InvalidArgument(
|
||||
"trainer_id %d is less than 0. Its "
|
||||
"valid range is [0, trainer_size)"));
|
||||
PADDLE_ENFORCE_LT(
|
||||
trainer_id, static_cast<int>(trainers.size()),
|
||||
platform::errors::OutOfRange("trainer_id %d is out of range. Its valid "
|
||||
"range is [0, trainer_size)",
|
||||
trainer_id));
|
||||
|
||||
int bkcl_comm_num = Attr<int>("bkcl_comm_num");
|
||||
int use_hierarchical_allreduce = Attr<bool>("use_hierarchical_allreduce");
|
||||
int inter_nranks = Attr<int>("hierarchical_allreduce_inter_nranks");
|
||||
int inter_trainer_id = -1;
|
||||
int exter_trainer_id = -1;
|
||||
|
||||
if (use_hierarchical_allreduce) {
|
||||
PADDLE_ENFORCE_GT(
|
||||
trainers.size(), 1,
|
||||
platform::errors::PreconditionNotMet(
|
||||
"The number of collective trainers %llu <= 1", trainers.size()));
|
||||
PADDLE_ENFORCE_GT(
|
||||
inter_nranks, 1,
|
||||
platform::errors::PreconditionNotMet(
|
||||
"inter_nranks %d <= 1 while in hierarchical allreduce mode",
|
||||
inter_nranks));
|
||||
PADDLE_ENFORCE_EQ(
|
||||
trainers.size() % inter_nranks, 0,
|
||||
platform::errors::PreconditionNotMet(
|
||||
"The number of trainers %llu mod inter_nranks %d is not equal 0",
|
||||
trainers.size(), inter_nranks));
|
||||
|
||||
inter_trainer_id = trainer_id % inter_nranks;
|
||||
|
||||
if (trainer_id % inter_nranks == 0) {
|
||||
exter_trainer_id = trainer_id / inter_nranks;
|
||||
}
|
||||
}
|
||||
|
||||
std::ostringstream ss;
|
||||
for (size_t i = 0; i < trainers.size(); i++) {
|
||||
ss << trainers[i] << ",";
|
||||
}
|
||||
|
||||
VLOG(1) << "trainer_id:" << trainer_id
|
||||
<< ", use_hierarchical_allreduce:" << use_hierarchical_allreduce
|
||||
<< ", bkcl_comm_num:" << bkcl_comm_num
|
||||
<< ", inter_nranks:" << inter_nranks
|
||||
<< ", inter_trainer_id:" << inter_trainer_id
|
||||
<< ", exter_trainer_id:" << exter_trainer_id
|
||||
<< ", trainers:" << ss.str();
|
||||
|
||||
int server_fd = -1;
|
||||
std::vector<BKCLUniqueId> bkcl_ids;
|
||||
bkcl_ids.resize(bkcl_comm_num);
|
||||
|
||||
/// 1. init flat
|
||||
std::function<std::string(size_t)> func = platform::GetFlatBKCLVarName;
|
||||
// broadcast unique id
|
||||
if (trainer_id == 0) {
|
||||
GenBKCLID(&bkcl_ids);
|
||||
|
||||
// server endpoints
|
||||
std::vector<std::string> flat_endpoints;
|
||||
flat_endpoints.insert(flat_endpoints.begin(), trainers.begin() + 1,
|
||||
trainers.end());
|
||||
platform::SendBroadCastCommID(flat_endpoints, &bkcl_ids);
|
||||
} else {
|
||||
server_fd = platform::CreateListenSocket(endpoint);
|
||||
platform::RecvBroadCastCommID(server_fd, endpoint, &bkcl_ids);
|
||||
}
|
||||
CopyBKCLIDToVar(bkcl_ids, func, scope);
|
||||
|
||||
/*TODO(liuyuhui) Baidu Kunlun Communication Library(BKCL) don't support
|
||||
hierarchical communication
|
||||
as NVIDIA Collective Communications Library(NCCL) in multi Nvidia GPU cards,
|
||||
and will support it later.
|
||||
*/
|
||||
// close socket server
|
||||
if (trainer_id != 0) {
|
||||
platform::CloseSocket(server_fd);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
class GenBKCLIdOpMaker : public framework::OpProtoAndCheckerMaker {
|
||||
public:
|
||||
void Make() override {
|
||||
AddOutput("BKCLID", "Raw variable contains a BKCL UniqueId instaces.");
|
||||
AddComment(R"DOC(
|
||||
GenBKCLId operator
|
||||
|
||||
For trainer 0: generate a new UniqueId and send it to all the other trainers.
|
||||
For trainer 1~n: start a gRPC server to get the UniqueId, once got, stop the server.
|
||||
)DOC");
|
||||
AddAttr<std::vector<std::string>>(
|
||||
"trainers",
|
||||
"['trainer0_ip:port', 'trainer1_ip:port', ...] "
|
||||
"list of all trainer endpoints")
|
||||
.SetDefault({});
|
||||
AddAttr<int>("trainer_id",
|
||||
"(int) "
|
||||
"The index of the trainer in distributed training.");
|
||||
AddAttr<int>("bkcl_comm_num",
|
||||
"(int default 1) "
|
||||
"The number of bkcl communicator num.")
|
||||
.SetDefault(1);
|
||||
AddAttr<bool>("use_hierarchical_allreduce",
|
||||
"(bool default false) "
|
||||
"Wheter to use hierarchical allreduce.")
|
||||
.SetDefault(false);
|
||||
AddAttr<int>("hierarchical_allreduce_inter_nranks",
|
||||
"(int default 1) "
|
||||
"Wheter to use hierarchical allreduce.")
|
||||
.SetDefault(-1);
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace operators
|
||||
} // namespace paddle
|
||||
|
||||
namespace ops = paddle::operators;
|
||||
|
||||
REGISTER_OPERATOR(gen_bkcl_id, ops::GenBKCLIdOp, ops::GenBKCLIdOpMaker);
|
||||
@ -0,0 +1,123 @@
|
||||
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import unittest
|
||||
import os
|
||||
import copy
|
||||
import sys
|
||||
sys.path.append("..")
|
||||
from launch_function_helper import wait, _find_free_port
|
||||
from multiprocessing import Pool, Process
|
||||
from threading import Thread
|
||||
|
||||
os.environ['GLOG_vmodule'] = str("gen_bkcl_id_op*=10,gen_comm_id*=10")
|
||||
|
||||
import paddle
|
||||
from paddle.fluid import core
|
||||
|
||||
paddle.enable_static()
|
||||
|
||||
|
||||
def run_gen_bkc_id(attr):
|
||||
bkcl_comm_num = attr['bkcl_comm_num']
|
||||
use_hallreduce = attr['use_hierarchical_allreduce']
|
||||
|
||||
startup_program = paddle.static.default_startup_program()
|
||||
main_program = paddle.static.default_main_program()
|
||||
|
||||
with paddle.static.program_guard(main_program, startup_program):
|
||||
bkcl_id_var = startup_program.global_block().create_var(
|
||||
name="BKCLID", persistable=True, type=core.VarDesc.VarType.RAW)
|
||||
|
||||
for i in range(1, bkcl_comm_num):
|
||||
startup_program.global_block().create_var(
|
||||
name="BKCLID_{}".format(i),
|
||||
persistable=True,
|
||||
type=core.VarDesc.VarType.RAW)
|
||||
|
||||
if use_hallreduce:
|
||||
for i in range(0, bkcl_comm_num):
|
||||
startup_program.global_block().create_var(
|
||||
name="Hierarchical_inter_BKCLID_{}".format(i),
|
||||
persistable=True,
|
||||
type=core.VarDesc.VarType.RAW)
|
||||
startup_program.global_block().create_var(
|
||||
name="Hierarchical_exter_BKCLID_{}".format(i),
|
||||
persistable=True,
|
||||
type=core.VarDesc.VarType.RAW)
|
||||
|
||||
startup_program.global_block().append_op(
|
||||
type="gen_bkcl_id",
|
||||
inputs={},
|
||||
outputs={"BKCLID": bkcl_id_var},
|
||||
attrs=attr)
|
||||
|
||||
place = paddle.CPUPlace()
|
||||
exe = paddle.static.Executor(place)
|
||||
exe.run(startup_program)
|
||||
|
||||
|
||||
class TestGenBKCLIdOp(unittest.TestCase):
|
||||
def setUp(self):
|
||||
try:
|
||||
self._dist_ut_port_0 = int(os.environ["PADDLE_DIST_UT_PORT"])
|
||||
except Exception as e:
|
||||
self._dist_ut_port_0 = _find_free_port(set())
|
||||
|
||||
def gen_bkcl_id(self, nranks=2):
|
||||
bkcl_comm_num = 1
|
||||
if nranks == 2:
|
||||
use_hallreduce = False
|
||||
hallreduce_inter_nranks = -1
|
||||
elif nranks == 4:
|
||||
use_hallreduce = True
|
||||
hallreduce_inter_nranks = 2
|
||||
|
||||
port = self._dist_ut_port_0
|
||||
trainers = []
|
||||
for i in range(nranks):
|
||||
trainers.append('127.0.0.1:{}'.format(port + i))
|
||||
|
||||
attr = {
|
||||
"trainers": trainers,
|
||||
"trainer_id": 0,
|
||||
"bkcl_comm_num": bkcl_comm_num,
|
||||
"use_hierarchical_allreduce": use_hallreduce,
|
||||
"hierarchical_allreduce_inter_nranks": hallreduce_inter_nranks,
|
||||
}
|
||||
|
||||
procs = []
|
||||
for i in range(nranks):
|
||||
attr['trainer_id'] = i
|
||||
# NOTE: multiprocessing cannot be covered by coverage
|
||||
p = Process(target=run_gen_bkc_id, args=(attr, ))
|
||||
p.start()
|
||||
procs.append(p)
|
||||
|
||||
wait(procs, timeout=120)
|
||||
|
||||
def test_flat(self):
|
||||
print(">>> test gen flat bkcl id")
|
||||
self.gen_bkcl_id(2)
|
||||
print("<<< end test gen flat bkcl id")
|
||||
print()
|
||||
|
||||
def test_hierarchical(self):
|
||||
print(">>> test gen hierarchical bkcl id")
|
||||
self.gen_bkcl_id(4)
|
||||
print("<<< end test gen hierarchical bkcl id")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Loading…
Reference in new issue