m53297601
/
Paddle
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fix fluid.embedding in Distributed Training (#25174)

Browse Source 
* test=develop, fix_embedding
fix_copy_if_different
Chengmo 5 years ago committed by GitHub
parent c701588b14
commit e85fcaa712
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 90 additions and 47 deletions
Show Stats Download Patch File Download Diff File

12
paddle/fluid/operators/distributed/parameter_prefetch.cc
Unescape View File

@ -209,16 +209,20 @@ void prefetchs(const std::vector<std::string>& id_var_names,
TableAndEndpoints tables;
for (auto& id_name : id_var_names) {
auto& id_tensor = scope.FindVar(id_name)->Get<framework::LoDTensor>();
auto* id_data = id_tensor.data<int64_t>();
auto* id_tensor =
scope.FindVar(id_name)->GetMutable<framework::LoDTensor>();
auto id_dims = id_tensor->dims();
id_tensor->Resize(framework::make_ddim(
{static_cast<int64_t>(id_dims[0] * id_dims[1]), 1}));
auto* id_data = id_tensor->data<int64_t>();
std::vector<int64_t> ids;
for (int64_t i = 0; i < id_tensor.numel(); ++i) {
for (int64_t i = 0; i < id_tensor->numel(); ++i) {
ids.push_back(id_data[i]);
ids_union.push_back(id_data[i]);
}
ids_group.push_back(ids);
ids_lods.push_back(id_tensor.lod());
ids_lods.push_back(id_tensor->lod());
}
std::unordered_set<int64_t> s(ids_union.begin(), ids_union.end());

39
paddle/fluid/operators/distributed_ops/distributed_lookup_table_op.cc
Unescape View File

@ -43,8 +43,6 @@ class DistributedLookupTableOp : public framework::OperatorWithKernel {
for (auto& ids_dim : ids_dims) {
PADDLE_ENFORCE_EQ(ids_dim.size(), 2,
"The dimension of the 'Ids' tensor must be 2.");
PADDLE_ENFORCE_EQ(ids_dim[1], 1,
"The last dimension of the 'Ids' tensor must be 1.");
}
auto lookup_tables =
@ -52,6 +50,8 @@ class DistributedLookupTableOp : public framework::OperatorWithKernel {
auto height_sections =
ctx->Attrs().Get<std::vector<int64_t>>("height_sections");
auto endpoints = ctx->Attrs().Get<std::vector<std::string>>("endpoints");
auto lookup_table_version =
ctx->Attrs().Get<std::string>("lookup_table_version");
PADDLE_ENFORCE(lookup_tables.size() == height_sections.size() &&
lookup_tables.size() == endpoints.size() &&
@ -62,7 +62,14 @@ class DistributedLookupTableOp : public framework::OperatorWithKernel {
auto outputs_dims = std::vector<framework::DDim>();
for (auto& ids_dim : ids_dims) {
outputs_dims.push_back(framework::make_ddim({ids_dim[0], table_dims[1]}));
if (lookup_table_version == "lookup_table") {
outputs_dims.push_back(
framework::make_ddim({ids_dim[0], table_dims[1]}));
} else if (lookup_table_version == "lookup_table_v2") {
outputs_dims.push_back(framework::make_ddim(
{static_cast<int64_t>(ids_dim[0]), static_cast<int64_t>(ids_dim[1]),
static_cast<int64_t>(table_dims[1])}));
}
}
ctx->SetOutputsDim("Outputs", outputs_dims);
@ -93,10 +100,30 @@ class DistributedLookupTableKernel : public framework::OpKernel<T> {
auto height_sections =
context.Attr<std::vector<int64_t>>("height_sections");
auto endpoints = context.Attr<std::vector<std::string>>("endpoints");
auto lookup_table_version =
context.Attr<std::string>("lookup_table_version");
operators::distributed::prefetchs(
id_names, out_names, embedding_name, false, lookup_tables, endpoints,
height_sections, context, context.scope());
if (lookup_table_version == "lookup_table_v2") {
auto& scope = context.scope();
auto emb_dim =
scope.FindVar(embedding_name)->Get<framework::LoDTensor>().dims()[1];
for (size_t i = 0; i < id_names.size(); ++i) {
auto* id_var = scope.FindVar(id_names[i]);
auto* out_var = scope.FindVar(out_names[i]);
auto* id_tensor = id_var->GetMutable<framework::LoDTensor>();
auto* out_tensor = out_var->GetMutable<framework::LoDTensor>();
auto id_dims = id_tensor->dims();
out_tensor->Resize(framework::make_ddim(
{static_cast<int64_t>(id_dims[0]), static_cast<int64_t>(id_dims[1]),
static_cast<int64_t>(emb_dim)}));
}
}
}
};
@ -134,6 +161,12 @@ class DistributedLookupTableOpMaker : public framework::OpProtoAndCheckerMaker {
AddAttr<int>("trainer_id", "trainer id from 0 ~ worker_num.").SetDefault(0);
AddAttr<std::string>(
"lookup_table_version",
"(string, default lookup_table) "
"To distinguish between different versions of embedding OP")
.SetDefault(std::string("lookup_table"));
AddAttr<int64_t>("padding_idx",
"(int64, default -1) "
"If the value is -1, it makes no effect to lookup. "

27
python/paddle/fluid/tests/unittests/dist_simnet_bow.py
Unescape View File

@ -92,8 +92,8 @@ def train_network(batch_size,
# query
q = fluid.layers.data(
name="query_ids", shape=[1], dtype="int64", lod_level=1)
## embedding
q_emb = fluid.layers.embedding(
# embedding
q_emb = fluid.embedding(
input=q,
is_distributed=is_distributed,
size=[dict_dim, emb_dim],
@ -104,10 +104,11 @@ def train_network(batch_size,
initializer=fluid.initializer.Constant(value=0.01),
name="__emb__"),
is_sparse=is_sparse)
## vsum
q_emb = fluid.layers.reshape(q_emb, [-1, emb_dim])
# vsum
q_sum = fluid.layers.sequence_pool(input=q_emb, pool_type='sum')
q_ss = fluid.layers.softsign(q_sum)
## fc layer after conv
# fc layer after conv
q_fc = fluid.layers.fc(
input=q_ss,
size=hid_dim,
@ -120,8 +121,8 @@ def train_network(batch_size,
# pt
pt = fluid.layers.data(
name="pos_title_ids", shape=[1], dtype="int64", lod_level=1)
## embedding
pt_emb = fluid.layers.embedding(
# embedding
pt_emb = fluid.embedding(
input=pt,
is_distributed=is_distributed,
size=[dict_dim, emb_dim],
@ -132,10 +133,11 @@ def train_network(batch_size,
initializer=fluid.initializer.Constant(value=0.01),
name="__emb__"),
is_sparse=is_sparse)
## vsum
pt_emb = fluid.layers.reshape(pt_emb, [-1, emb_dim])
# vsum
pt_sum = fluid.layers.sequence_pool(input=pt_emb, pool_type='sum')
pt_ss = fluid.layers.softsign(pt_sum)
## fc layer
# fc layer
pt_fc = fluid.layers.fc(
input=pt_ss,
size=hid_dim,
@ -147,8 +149,8 @@ def train_network(batch_size,
# nt
nt = fluid.layers.data(
name="neg_title_ids", shape=[1], dtype="int64", lod_level=1)
## embedding
nt_emb = fluid.layers.embedding(
# embedding
nt_emb = fluid.embedding(
input=nt,
is_distributed=is_distributed,
size=[dict_dim, emb_dim],
@ -159,10 +161,11 @@ def train_network(batch_size,
initializer=fluid.initializer.Constant(value=0.01),
name="__emb__"),
is_sparse=is_sparse)
## vsum
nt_emb = fluid.layers.reshape(nt_emb, [-1, emb_dim])
# vsum
nt_sum = fluid.layers.sequence_pool(input=nt_emb, pool_type='sum')
nt_ss = fluid.layers.softsign(nt_sum)
## fc layer
# fc layer
nt_fc = fluid.layers.fc(
input=nt_ss,
size=hid_dim,

6
python/paddle/fluid/tests/unittests/test_dist_simnet_bow.py
Unescape View File

@ -107,7 +107,7 @@ class TestDistSimnetBow2x2LookupTableSync(TestDistBase):
def test_simnet_bow(self):
need_envs = {
"IS_DISTRIBUTED": '1',
"IS_DISTRIBUTED": '0',
"IS_SPARSE": '1',
'IS_SELF_CONTAINED_LR': '1'
}
@ -126,7 +126,7 @@ class TestDistSimnetBow2x2LookupTableAsync(TestDistBase):
def test_simnet_bow(self):
need_envs = {
"IS_DISTRIBUTED": '1',
"IS_DISTRIBUTED": '0',
"IS_SPARSE": '1',
'IS_SELF_CONTAINED_LR': '1'
}
@ -145,7 +145,7 @@ class TestDistSimnetBow2x2LookupTableNotContainLRSync(TestDistBase):
def test_simnet_bow(self):
need_envs = {
"IS_DISTRIBUTED": '1',
"IS_DISTRIBUTED": '0',
"IS_SPARSE": '1',
'IS_SELF_CONTAINED_LR': '0'
}

19
python/paddle/fluid/transpiler/distribute_transpiler.py
Unescape View File

@ -50,8 +50,8 @@ from .details import delete_ops, find_op_by_output_arg
from ..distribute_lookup_table import find_distributed_lookup_table
from . import collective
LOOKUP_TABLE_TYPE = "lookup_table"
LOOKUP_TABLE_GRAD_TYPE = "lookup_table_grad"
LOOKUP_TABLE_TYPE = ["lookup_table", "lookup_table_v2"]
LOOKUP_TABLE_GRAD_TYPE = ["lookup_table_grad", "lookup_table_v2_grad"]
OP_NAME_SCOPE = "op_namescope"
CLIP_OP_NAME_SCOPE = "@CLIP"
OP_ROLE_VAR_ATTR_NAME = core.op_proto_and_checker_maker.kOpRoleVarAttrName()
@ -449,7 +449,7 @@ class DistributeTranspiler(object):
def _get_all_remote_sparse_update_op(self, main_program):
sparse_update_ops = []
sparse_update_op_types = ["lookup_table", "nce"]
sparse_update_op_types = ["lookup_table", "nce", "lookup_table_v2"]
for op in main_program.global_block().ops:
if op.type in sparse_update_op_types and op.attr(
'remote_prefetch') is True:
@ -479,7 +479,7 @@ class DistributeTranspiler(object):
ops.append(op)
used_ops.append(idx)
if op_type == "lookup_table":
if op_type in LOOKUP_TABLE_TYPE:
all_ops = program.global_block().ops
op_idxs = [all_ops.index(op) for op in ops]
inputs = [
@ -525,7 +525,8 @@ class DistributeTranspiler(object):
"height_sections": height_sections,
"endpoints": endpoints,
"padding_idx": padding_idx,
"trainer_id": self.trainer_id
"trainer_id": self.trainer_id,
"lookup_table_version": op_type
})
else:
raise ValueError(
@ -613,10 +614,12 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler
)
assert trainers_num > self.config.hierarchical_allreduce_inter_nranks, \
"trainers_num:{} < hierarchical_allreduce_inter_nranks:{}".format(trainers_num, self.config.hierarchical_allreduce_inter_nranks)
"trainers_num:{} < hierarchical_allreduce_inter_nranks:{}".format(
trainers_num, self.config.hierarchical_allreduce_inter_nranks)
assert trainers_num % self.config.hierarchical_allreduce_inter_nranks == 0, \
"trainers_num:{} mod hierarchical_allreduce_inter_nranks:{} != 0".format(trainers_num, self.config.hierarchical_allreduce_inter_nranks)
"trainers_num:{} mod hierarchical_allreduce_inter_nranks:{} != 0".format(
trainers_num, self.config.hierarchical_allreduce_inter_nranks)
self.origin_program._hierarchical_allreduce_inter_nranks = \
int(self.config.hierarchical_allreduce_inter_nranks)
@ -782,7 +785,7 @@ WIKI: https://github.com/PaddlePaddle/Fleet/blob/develop/markdown_doc/transpiler
decay_dummy_output = program.global_block().create_var(
name=framework.generate_control_dev_var_name())
if self.config.runtime_split_send_recv:
## async mode, using communicator to merge and send
# async mode, using communicator to merge and send
send_varnames = [self.counter_var.name]
else:
send_varnames = []

Write Preview
Loading…
Cancel
Save