Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into add_cudnn_5_support
liuhongyu
6 years ago
commit
773dc73fbf
@ -0,0 +1,117 @@
|
|||||||
|
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License. */
|
||||||
|
|
||||||
|
#include "paddle/fluid/framework/op_registry.h"
|
||||||
|
#include "paddle/fluid/framework/tensor_util.h"
|
||||||
|
|
||||||
|
namespace paddle {
|
||||||
|
namespace operators {
|
||||||
|
|
||||||
|
class GetTensorFromSelectedRowsOp : public framework::OperatorWithKernel {
|
||||||
|
public:
|
||||||
|
using framework::OperatorWithKernel::OperatorWithKernel;
|
||||||
|
|
||||||
|
void InferShape(framework::InferShapeContext *ctx) const override {
|
||||||
|
PADDLE_ENFORCE(ctx->HasInput("X"),
|
||||||
|
"GetTensorFromSelectedRowsOp must has input X.");
|
||||||
|
PADDLE_ENFORCE(ctx->HasOutput("Out"),
|
||||||
|
"GetTensorFromSelectedRowsOp must has output Out.");
|
||||||
|
PADDLE_ENFORCE(
|
||||||
|
ctx->GetInputsVarType("X").front() ==
|
||||||
|
framework::proto::VarType::SELECTED_ROWS,
|
||||||
|
"The input X's type should be SelectedRows, but the received is %s",
|
||||||
|
ctx->Inputs("X").front(), ctx->GetInputsVarType("X").front());
|
||||||
|
PADDLE_ENFORCE(
|
||||||
|
ctx->GetOutputsVarType("Out").front() ==
|
||||||
|
framework::proto::VarType::LOD_TENSOR,
|
||||||
|
"The output Out's type should be LoDTensor, but the received is %s",
|
||||||
|
ctx->Outputs("Out").front(), ctx->GetOutputsVarType("Out").front());
|
||||||
|
|
||||||
|
ctx->SetOutputDim("Out", ctx->GetInputDim("X"));
|
||||||
|
}
|
||||||
|
|
||||||
|
protected:
|
||||||
|
framework::OpKernelType GetExpectedKernelType(
|
||||||
|
const framework::ExecutionContext &ctx) const override {
|
||||||
|
return framework::OpKernelType(
|
||||||
|
framework::GetDataTypeOfVar(ctx.InputVar("X")), ctx.device_context());
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
class GetTensorFromSelectedRowsKernel {
|
||||||
|
public:
|
||||||
|
void operator()(const framework::ExecutionContext &ctx) const {
|
||||||
|
auto *x = ctx.Input<framework::SelectedRows>("X");
|
||||||
|
auto *out = ctx.Output<framework::LoDTensor>("Out");
|
||||||
|
|
||||||
|
out->Resize(x->value().dims());
|
||||||
|
out->mutable_data(ctx.GetPlace(), x->value().type());
|
||||||
|
framework::TensorCopy(x->value(), ctx.GetPlace(), ctx.device_context(),
|
||||||
|
out);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
class GetTensorFromSelectedRowsOpProtoMaker
|
||||||
|
: public framework::OpProtoAndCheckerMaker {
|
||||||
|
public:
|
||||||
|
void Make() override {
|
||||||
|
AddInput("X", "The input type is SelectedRows.");
|
||||||
|
AddOutput("Out", "The output type is LoDTensor.");
|
||||||
|
AddComment(
|
||||||
|
R"DOC(
|
||||||
|
GetTensorFromSelectedRows Operator
|
||||||
|
|
||||||
|
GetTensorFromSelectedRows is used to get the tensor from SelectedRows.
|
||||||
|
|
||||||
|
)DOC");
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
class GetTensorFromSelectedRowsOpVarTypeInference
|
||||||
|
: public framework::VarTypeInference {
|
||||||
|
public:
|
||||||
|
void operator()(const framework::OpDesc &op_desc,
|
||||||
|
framework::BlockDesc *block) const final {
|
||||||
|
auto out_var_name = op_desc.Output("Out").front();
|
||||||
|
auto in_var_name = op_desc.Input("X").front();
|
||||||
|
|
||||||
|
auto out_var = block->FindRecursiveOrCreateVar(out_var_name);
|
||||||
|
auto in_var = block->FindRecursiveOrCreateVar(in_var_name);
|
||||||
|
out_var.SetType(framework::proto::VarType::LOD_TENSOR);
|
||||||
|
out_var.SetDataType(in_var.GetDataType());
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
} // namespace operators
|
||||||
|
} // namespace paddle
|
||||||
|
|
||||||
|
namespace ops = paddle::operators;
|
||||||
|
REGISTER_OPERATOR(get_tensor_from_selected_rows,
|
||||||
|
ops::GetTensorFromSelectedRowsOp,
|
||||||
|
ops::GetTensorFromSelectedRowsOpProtoMaker,
|
||||||
|
ops::GetTensorFromSelectedRowsOpVarTypeInference);
|
||||||
|
|
||||||
|
REGISTER_OP_CPU_KERNEL_FUNCTOR(get_tensor_from_selected_rows, float,
|
||||||
|
ops::GetTensorFromSelectedRowsKernel, double,
|
||||||
|
ops::GetTensorFromSelectedRowsKernel, int,
|
||||||
|
ops::GetTensorFromSelectedRowsKernel, int64_t,
|
||||||
|
ops::GetTensorFromSelectedRowsKernel);
|
||||||
|
|
||||||
|
#ifdef PADDLE_WITH_CUDA
|
||||||
|
REGISTER_OP_CUDA_KERNEL_FUNCTOR(get_tensor_from_selected_rows, float,
|
||||||
|
ops::GetTensorFromSelectedRowsKernel, double,
|
||||||
|
ops::GetTensorFromSelectedRowsKernel, int,
|
||||||
|
ops::GetTensorFromSelectedRowsKernel, int64_t,
|
||||||
|
ops::GetTensorFromSelectedRowsKernel);
|
||||||
|
#endif
|
||||||
@ -0,0 +1,72 @@
|
|||||||
|
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License. */
|
||||||
|
|
||||||
|
#include "paddle/fluid/operators/merge_selected_rows_op.h"
|
||||||
|
|
||||||
|
namespace paddle {
|
||||||
|
namespace operators {
|
||||||
|
|
||||||
|
class MergeSelectedRowsOp : public framework::OperatorWithKernel {
|
||||||
|
public:
|
||||||
|
using framework::OperatorWithKernel::OperatorWithKernel;
|
||||||
|
|
||||||
|
void InferShape(framework::InferShapeContext* ctx) const override {
|
||||||
|
PADDLE_ENFORCE(ctx->HasInput("X"),
|
||||||
|
"Input(X) of MergeSelectedRowsOp should not be null.");
|
||||||
|
PADDLE_ENFORCE(ctx->HasOutput("Out"),
|
||||||
|
"Output(Out) of MergeSelectedRowsOp should not be null.");
|
||||||
|
ctx->ShareDim("X", /*->*/ "Out");
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
class MergeSelectedRowsOpMaker : public framework::OpProtoAndCheckerMaker {
|
||||||
|
public:
|
||||||
|
void Make() override {
|
||||||
|
AddInput("X",
|
||||||
|
"The input type is SelectedRows, and the selected rows may be "
|
||||||
|
"duplicated.");
|
||||||
|
AddOutput("Out",
|
||||||
|
"The output type is SelectedRows, and the selected rows are not "
|
||||||
|
"duplicated.");
|
||||||
|
AddComment(
|
||||||
|
R"DOC(
|
||||||
|
MergeSelectedRows Operator.
|
||||||
|
|
||||||
|
MergeSelectedRows is used to merge the duplicated rows of the input.
|
||||||
|
)DOC");
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
class MergeSelectedRowsOpInferVarType
|
||||||
|
: public framework::PassInDtypeAndVarTypeToOutput {
|
||||||
|
protected:
|
||||||
|
std::unordered_map<std::string, std::string> GetInputOutputWithSameType()
|
||||||
|
const override {
|
||||||
|
return std::unordered_map<std::string, std::string>{{"X", /*->*/ "Out"}};
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
} // namespace operators
|
||||||
|
} // namespace paddle
|
||||||
|
|
||||||
|
namespace ops = paddle::operators;
|
||||||
|
namespace plat = paddle::platform;
|
||||||
|
REGISTER_OPERATOR(merge_selected_rows, ops::MergeSelectedRowsOp,
|
||||||
|
ops::MergeSelectedRowsOpMaker,
|
||||||
|
ops::MergeSelectedRowsOpInferVarType);
|
||||||
|
|
||||||
|
REGISTER_OP_CPU_KERNEL(
|
||||||
|
merge_selected_rows,
|
||||||
|
ops::MergeSelectedRowsKernel<plat::CPUDeviceContext, float>,
|
||||||
|
ops::MergeSelectedRowsKernel<plat::CPUDeviceContext, double>);
|
||||||
@ -0,0 +1,23 @@
|
|||||||
|
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License. */
|
||||||
|
|
||||||
|
#include "paddle/fluid/operators/merge_selected_rows_op.h"
|
||||||
|
|
||||||
|
namespace ops = paddle::operators;
|
||||||
|
namespace plat = paddle::platform;
|
||||||
|
|
||||||
|
REGISTER_OP_CUDA_KERNEL(
|
||||||
|
merge_selected_rows,
|
||||||
|
ops::MergeSelectedRowsKernel<plat::CUDADeviceContext, float>,
|
||||||
|
ops::MergeSelectedRowsKernel<plat::CUDADeviceContext, double>);
|
||||||
@ -0,0 +1,36 @@
|
|||||||
|
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License. */
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
#include <string>
|
||||||
|
#include "paddle/fluid/framework/op_registry.h"
|
||||||
|
#include "paddle/fluid/operators/math/selected_rows_functor.h"
|
||||||
|
|
||||||
|
namespace paddle {
|
||||||
|
namespace operators {
|
||||||
|
|
||||||
|
template <typename DeviceContext, typename T>
|
||||||
|
class MergeSelectedRowsKernel : public framework::OpKernel<T> {
|
||||||
|
public:
|
||||||
|
void Compute(const framework::ExecutionContext& context) const override {
|
||||||
|
auto* x = context.Input<framework::SelectedRows>("X");
|
||||||
|
auto* out = context.Output<framework::SelectedRows>("Out");
|
||||||
|
|
||||||
|
math::scatter::MergeAdd<DeviceContext, T> merge_func;
|
||||||
|
merge_func(context.template device_context<DeviceContext>(), *x, out);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
} // namespace operators
|
||||||
|
} // namespace paddle
|
||||||
@ -0,0 +1,37 @@
|
|||||||
|
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License. */
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
#include <sstream>
|
||||||
|
#include <string>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
namespace paddle {
|
||||||
|
namespace string {
|
||||||
|
|
||||||
|
static inline std::vector<std::string> Split(std::string const& original,
|
||||||
|
char separator) {
|
||||||
|
std::vector<std::string> results;
|
||||||
|
std::string token;
|
||||||
|
std::istringstream is(original);
|
||||||
|
while (std::getline(is, token, separator)) {
|
||||||
|
if (!token.empty()) {
|
||||||
|
results.push_back(token);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace string
|
||||||
|
} // namespace paddle
|
||||||
@ -0,0 +1,28 @@
|
|||||||
|
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
#include "paddle/fluid/string/split.h"
|
||||||
|
|
||||||
|
#include <string>
|
||||||
|
|
||||||
|
#include "gtest/gtest.h"
|
||||||
|
|
||||||
|
TEST(StringSplit, StringSplit) {
|
||||||
|
std::string to_split = "0,1,2,3,4,5";
|
||||||
|
int i = 0;
|
||||||
|
for (auto s : paddle::string::Split(to_split, ',')) {
|
||||||
|
EXPECT_EQ(atoi(s.c_str()), i);
|
||||||
|
i++;
|
||||||
|
}
|
||||||
|
}
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue