Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into add_cudnn_5_support
liuhongyu
6 years ago
commit
773dc73fbf
@ -0,0 +1,117 @@
|
||||
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#include "paddle/fluid/framework/op_registry.h"
|
||||
#include "paddle/fluid/framework/tensor_util.h"
|
||||
|
||||
namespace paddle {
|
||||
namespace operators {
|
||||
|
||||
class GetTensorFromSelectedRowsOp : public framework::OperatorWithKernel {
|
||||
public:
|
||||
using framework::OperatorWithKernel::OperatorWithKernel;
|
||||
|
||||
void InferShape(framework::InferShapeContext *ctx) const override {
|
||||
PADDLE_ENFORCE(ctx->HasInput("X"),
|
||||
"GetTensorFromSelectedRowsOp must has input X.");
|
||||
PADDLE_ENFORCE(ctx->HasOutput("Out"),
|
||||
"GetTensorFromSelectedRowsOp must has output Out.");
|
||||
PADDLE_ENFORCE(
|
||||
ctx->GetInputsVarType("X").front() ==
|
||||
framework::proto::VarType::SELECTED_ROWS,
|
||||
"The input X's type should be SelectedRows, but the received is %s",
|
||||
ctx->Inputs("X").front(), ctx->GetInputsVarType("X").front());
|
||||
PADDLE_ENFORCE(
|
||||
ctx->GetOutputsVarType("Out").front() ==
|
||||
framework::proto::VarType::LOD_TENSOR,
|
||||
"The output Out's type should be LoDTensor, but the received is %s",
|
||||
ctx->Outputs("Out").front(), ctx->GetOutputsVarType("Out").front());
|
||||
|
||||
ctx->SetOutputDim("Out", ctx->GetInputDim("X"));
|
||||
}
|
||||
|
||||
protected:
|
||||
framework::OpKernelType GetExpectedKernelType(
|
||||
const framework::ExecutionContext &ctx) const override {
|
||||
return framework::OpKernelType(
|
||||
framework::GetDataTypeOfVar(ctx.InputVar("X")), ctx.device_context());
|
||||
}
|
||||
};
|
||||
|
||||
class GetTensorFromSelectedRowsKernel {
|
||||
public:
|
||||
void operator()(const framework::ExecutionContext &ctx) const {
|
||||
auto *x = ctx.Input<framework::SelectedRows>("X");
|
||||
auto *out = ctx.Output<framework::LoDTensor>("Out");
|
||||
|
||||
out->Resize(x->value().dims());
|
||||
out->mutable_data(ctx.GetPlace(), x->value().type());
|
||||
framework::TensorCopy(x->value(), ctx.GetPlace(), ctx.device_context(),
|
||||
out);
|
||||
}
|
||||
};
|
||||
|
||||
class GetTensorFromSelectedRowsOpProtoMaker
|
||||
: public framework::OpProtoAndCheckerMaker {
|
||||
public:
|
||||
void Make() override {
|
||||
AddInput("X", "The input type is SelectedRows.");
|
||||
AddOutput("Out", "The output type is LoDTensor.");
|
||||
AddComment(
|
||||
R"DOC(
|
||||
GetTensorFromSelectedRows Operator
|
||||
|
||||
GetTensorFromSelectedRows is used to get the tensor from SelectedRows.
|
||||
|
||||
)DOC");
|
||||
}
|
||||
};
|
||||
|
||||
class GetTensorFromSelectedRowsOpVarTypeInference
|
||||
: public framework::VarTypeInference {
|
||||
public:
|
||||
void operator()(const framework::OpDesc &op_desc,
|
||||
framework::BlockDesc *block) const final {
|
||||
auto out_var_name = op_desc.Output("Out").front();
|
||||
auto in_var_name = op_desc.Input("X").front();
|
||||
|
||||
auto out_var = block->FindRecursiveOrCreateVar(out_var_name);
|
||||
auto in_var = block->FindRecursiveOrCreateVar(in_var_name);
|
||||
out_var.SetType(framework::proto::VarType::LOD_TENSOR);
|
||||
out_var.SetDataType(in_var.GetDataType());
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace operators
|
||||
} // namespace paddle
|
||||
|
||||
namespace ops = paddle::operators;
|
||||
REGISTER_OPERATOR(get_tensor_from_selected_rows,
|
||||
ops::GetTensorFromSelectedRowsOp,
|
||||
ops::GetTensorFromSelectedRowsOpProtoMaker,
|
||||
ops::GetTensorFromSelectedRowsOpVarTypeInference);
|
||||
|
||||
REGISTER_OP_CPU_KERNEL_FUNCTOR(get_tensor_from_selected_rows, float,
|
||||
ops::GetTensorFromSelectedRowsKernel, double,
|
||||
ops::GetTensorFromSelectedRowsKernel, int,
|
||||
ops::GetTensorFromSelectedRowsKernel, int64_t,
|
||||
ops::GetTensorFromSelectedRowsKernel);
|
||||
|
||||
#ifdef PADDLE_WITH_CUDA
|
||||
REGISTER_OP_CUDA_KERNEL_FUNCTOR(get_tensor_from_selected_rows, float,
|
||||
ops::GetTensorFromSelectedRowsKernel, double,
|
||||
ops::GetTensorFromSelectedRowsKernel, int,
|
||||
ops::GetTensorFromSelectedRowsKernel, int64_t,
|
||||
ops::GetTensorFromSelectedRowsKernel);
|
||||
#endif
|
||||
@ -0,0 +1,72 @@
|
||||
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#include "paddle/fluid/operators/merge_selected_rows_op.h"
|
||||
|
||||
namespace paddle {
|
||||
namespace operators {
|
||||
|
||||
class MergeSelectedRowsOp : public framework::OperatorWithKernel {
|
||||
public:
|
||||
using framework::OperatorWithKernel::OperatorWithKernel;
|
||||
|
||||
void InferShape(framework::InferShapeContext* ctx) const override {
|
||||
PADDLE_ENFORCE(ctx->HasInput("X"),
|
||||
"Input(X) of MergeSelectedRowsOp should not be null.");
|
||||
PADDLE_ENFORCE(ctx->HasOutput("Out"),
|
||||
"Output(Out) of MergeSelectedRowsOp should not be null.");
|
||||
ctx->ShareDim("X", /*->*/ "Out");
|
||||
}
|
||||
};
|
||||
|
||||
class MergeSelectedRowsOpMaker : public framework::OpProtoAndCheckerMaker {
|
||||
public:
|
||||
void Make() override {
|
||||
AddInput("X",
|
||||
"The input type is SelectedRows, and the selected rows may be "
|
||||
"duplicated.");
|
||||
AddOutput("Out",
|
||||
"The output type is SelectedRows, and the selected rows are not "
|
||||
"duplicated.");
|
||||
AddComment(
|
||||
R"DOC(
|
||||
MergeSelectedRows Operator.
|
||||
|
||||
MergeSelectedRows is used to merge the duplicated rows of the input.
|
||||
)DOC");
|
||||
}
|
||||
};
|
||||
|
||||
class MergeSelectedRowsOpInferVarType
|
||||
: public framework::PassInDtypeAndVarTypeToOutput {
|
||||
protected:
|
||||
std::unordered_map<std::string, std::string> GetInputOutputWithSameType()
|
||||
const override {
|
||||
return std::unordered_map<std::string, std::string>{{"X", /*->*/ "Out"}};
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace operators
|
||||
} // namespace paddle
|
||||
|
||||
namespace ops = paddle::operators;
|
||||
namespace plat = paddle::platform;
|
||||
REGISTER_OPERATOR(merge_selected_rows, ops::MergeSelectedRowsOp,
|
||||
ops::MergeSelectedRowsOpMaker,
|
||||
ops::MergeSelectedRowsOpInferVarType);
|
||||
|
||||
REGISTER_OP_CPU_KERNEL(
|
||||
merge_selected_rows,
|
||||
ops::MergeSelectedRowsKernel<plat::CPUDeviceContext, float>,
|
||||
ops::MergeSelectedRowsKernel<plat::CPUDeviceContext, double>);
|
||||
@ -0,0 +1,23 @@
|
||||
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#include "paddle/fluid/operators/merge_selected_rows_op.h"
|
||||
|
||||
namespace ops = paddle::operators;
|
||||
namespace plat = paddle::platform;
|
||||
|
||||
REGISTER_OP_CUDA_KERNEL(
|
||||
merge_selected_rows,
|
||||
ops::MergeSelectedRowsKernel<plat::CUDADeviceContext, float>,
|
||||
ops::MergeSelectedRowsKernel<plat::CUDADeviceContext, double>);
|
||||
@ -0,0 +1,36 @@
|
||||
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#pragma once
|
||||
#include <string>
|
||||
#include "paddle/fluid/framework/op_registry.h"
|
||||
#include "paddle/fluid/operators/math/selected_rows_functor.h"
|
||||
|
||||
namespace paddle {
|
||||
namespace operators {
|
||||
|
||||
template <typename DeviceContext, typename T>
|
||||
class MergeSelectedRowsKernel : public framework::OpKernel<T> {
|
||||
public:
|
||||
void Compute(const framework::ExecutionContext& context) const override {
|
||||
auto* x = context.Input<framework::SelectedRows>("X");
|
||||
auto* out = context.Output<framework::SelectedRows>("Out");
|
||||
|
||||
math::scatter::MergeAdd<DeviceContext, T> merge_func;
|
||||
merge_func(context.template device_context<DeviceContext>(), *x, out);
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace operators
|
||||
} // namespace paddle
|
||||
@ -0,0 +1,37 @@
|
||||
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#pragma once
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace paddle {
|
||||
namespace string {
|
||||
|
||||
static inline std::vector<std::string> Split(std::string const& original,
|
||||
char separator) {
|
||||
std::vector<std::string> results;
|
||||
std::string token;
|
||||
std::istringstream is(original);
|
||||
while (std::getline(is, token, separator)) {
|
||||
if (!token.empty()) {
|
||||
results.push_back(token);
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
} // namespace string
|
||||
} // namespace paddle
|
||||
@ -0,0 +1,28 @@
|
||||
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "paddle/fluid/string/split.h"
|
||||
|
||||
#include <string>
|
||||
|
||||
#include "gtest/gtest.h"
|
||||
|
||||
TEST(StringSplit, StringSplit) {
|
||||
std::string to_split = "0,1,2,3,4,5";
|
||||
int i = 0;
|
||||
for (auto s : paddle::string::Split(to_split, ',')) {
|
||||
EXPECT_EQ(atoi(s.c_str()), i);
|
||||
i++;
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue