Fix the correctness of async mode at distributed training (#18863)
* fix correctness of the communicator * fix a bug in send thread when sending var context is empty, test=develop * add lookup_table_prefetch_op and prefetch optimize, test=develop * remove remote prefetch GPU supported * word2vec force with CPU, test=develop * test dist remote lookup table force with CPU, test=developassert
tangwei12
6 years ago
committed by
GitHub
parent
61389ae5aa
commit
65c7368400
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,166 @@
|
||||
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#include <algorithm>
|
||||
|
||||
#include "paddle/fluid/framework/data_type.h"
|
||||
#include "paddle/fluid/framework/op_registry.h"
|
||||
#include "paddle/fluid/operators/distributed/parameter_prefetch.h"
|
||||
#include "paddle/fluid/operators/math/math_function.h"
|
||||
|
||||
namespace paddle {
|
||||
namespace operators {
|
||||
|
||||
class DistributedLookupTableOp : public framework::OperatorWithKernel {
|
||||
public:
|
||||
using framework::OperatorWithKernel::OperatorWithKernel;
|
||||
|
||||
void InferShape(framework::InferShapeContext *ctx) const override {
|
||||
PADDLE_ENFORCE(ctx->HasInputs("Ids"),
|
||||
"Input(Ids) of LookupTableOp should not be null.");
|
||||
PADDLE_ENFORCE(ctx->HasInput("W"),
|
||||
"Input(W) of LookupTableOp should not be null.");
|
||||
PADDLE_ENFORCE(ctx->HasOutputs("Outputs"),
|
||||
"Output(Outs) of LookupTableOp should not be null.");
|
||||
|
||||
auto ids_dims = ctx->GetInputsDim("Ids");
|
||||
auto table_dims = ctx->GetInputDim("W");
|
||||
|
||||
PADDLE_ENFORCE_EQ(table_dims.size(), 2,
|
||||
"Only 2 dimensions of the 'Embedding' is supported.");
|
||||
|
||||
for (auto &ids_dim : ids_dims) {
|
||||
PADDLE_ENFORCE_EQ(ids_dim.size(), 2,
|
||||
"The dimension of the 'Ids' tensor must be 2.");
|
||||
PADDLE_ENFORCE_EQ(ids_dim[1], 1,
|
||||
"The last dimension of the 'Ids' tensor must be 1.");
|
||||
}
|
||||
|
||||
auto lookup_tables =
|
||||
ctx->Attrs().Get<std::vector<std::string>>("table_names");
|
||||
auto height_sections =
|
||||
ctx->Attrs().Get<std::vector<int64_t>>("height_sections");
|
||||
auto endpoints = ctx->Attrs().Get<std::vector<std::string>>("endpoints");
|
||||
|
||||
PADDLE_ENFORCE(lookup_tables.size() == height_sections.size() &&
|
||||
lookup_tables.size() == endpoints.size() &&
|
||||
lookup_tables.size() != 0,
|
||||
"Attrs lookup_tables/height_sections/endpoints must have "
|
||||
"save size and can not be 0.");
|
||||
|
||||
auto outputs_dims = std::vector<framework::DDim>();
|
||||
|
||||
for (auto &ids_dim : ids_dims) {
|
||||
outputs_dims.push_back(framework::make_ddim({ids_dim[0], table_dims[1]}));
|
||||
}
|
||||
|
||||
ctx->SetOutputsDim("Outputs", outputs_dims);
|
||||
ctx->ShareLoD("Ids", /*->*/ "Outputs");
|
||||
}
|
||||
|
||||
protected:
|
||||
framework::OpKernelType GetExpectedKernelType(
|
||||
const framework::ExecutionContext &ctx) const override {
|
||||
auto data_type = framework::GetDataTypeOfVar(ctx.InputVar("W"));
|
||||
return framework::OpKernelType(data_type, ctx.device_context());
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
class DistributedLookupTableKernel : public framework::OpKernel<T> {
|
||||
public:
|
||||
void Compute(const framework::ExecutionContext &context) const override {
|
||||
auto ids_vars = context.MultiInputVar("Ids");
|
||||
auto emb_vars = context.MultiOutput<framework::Tensor>("Embeddings");
|
||||
|
||||
auto id_names = context.Inputs("Ids");
|
||||
auto embedding_name = context.Inputs("W").front();
|
||||
auto out_names = context.Outputs("Outputs");
|
||||
|
||||
auto lookup_tables = context.Attr<std::vector<std::string>>("table_names");
|
||||
auto height_sections =
|
||||
context.Attr<std::vector<int64_t>>("height_sections");
|
||||
auto endpoints = context.Attr<std::vector<std::string>>("endpoints");
|
||||
|
||||
operators::distributed::prefetchs(
|
||||
id_names, out_names, embedding_name, false, lookup_tables, endpoints,
|
||||
height_sections, context, context.scope());
|
||||
}
|
||||
};
|
||||
|
||||
class DistributedLookupTableOpMaker : public framework::OpProtoAndCheckerMaker {
|
||||
public:
|
||||
void Make() override {
|
||||
AddInput("Ids",
|
||||
"(LoDTensor) Ids's type should be LoDTensor"
|
||||
"THe ids to be looked up in W.")
|
||||
.AsDuplicable();
|
||||
|
||||
AddInput("W",
|
||||
"(Tensor) The input represents embedding tensors, "
|
||||
"which is a learnable parameter.");
|
||||
|
||||
AddOutput("Outputs",
|
||||
"(LoDTensor) The lookup results, which have the same type as W.")
|
||||
.AsDuplicable();
|
||||
|
||||
AddAttr<std::vector<std::string>>(
|
||||
"table_names",
|
||||
"(string vector, such as emb_block0, emb_block1)"
|
||||
"Server endpoints in the order of input variables for mapping")
|
||||
.SetDefault({""});
|
||||
|
||||
AddAttr<std::vector<int64_t>>("height_sections",
|
||||
"Height for each output SelectedRows.")
|
||||
.SetDefault(std::vector<int64_t>({}));
|
||||
|
||||
AddAttr<std::vector<std::string>>(
|
||||
"endpoints",
|
||||
"(string vector, default 127.0.0.1:6164)"
|
||||
"Server endpoints in the order of input variables for mapping")
|
||||
.SetDefault({"127.0.0.1:6164"});
|
||||
|
||||
AddAttr<int>("trainer_id", "trainer id from 0 ~ worker_num.").SetDefault(0);
|
||||
|
||||
AddAttr<int64_t>("padding_idx",
|
||||
"(int64, default -1) "
|
||||
"If the value is -1, it makes no effect to lookup. "
|
||||
"Otherwise the given value indicates padding the output "
|
||||
"with zeros whenever lookup encounters it in Ids.")
|
||||
.SetDefault(distributed::kNoPadding);
|
||||
|
||||
AddComment(R"DOC(
|
||||
Lookup Tablel Prefetch Operator.
|
||||
|
||||
This operator is used to perform lookup on parameter W,
|
||||
then concatenated into a sparse tensor.
|
||||
|
||||
The type of Ids(Input) is SelectedRows, the rows of Ids contains
|
||||
the ids to be looked up in W;
|
||||
if the Id is not in the sparse table, this operator will return a
|
||||
random value and set the value into the table for the next looking up.
|
||||
|
||||
)DOC");
|
||||
}
|
||||
};
|
||||
} // namespace operators
|
||||
} // namespace paddle
|
||||
|
||||
namespace ops = paddle::operators;
|
||||
|
||||
REGISTER_OPERATOR(distributed_lookup_table, ops::DistributedLookupTableOp,
|
||||
ops::DistributedLookupTableOpMaker);
|
||||
|
||||
REGISTER_OP_CPU_KERNEL(distributed_lookup_table,
|
||||
ops::DistributedLookupTableKernel<float>);
|
||||
Loading…
Reference in new issue