From ab08575adf4c13a874aeb7cb1ad7b3904ba45f82 Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Fri, 11 Aug 2017 11:03:26 -0700 Subject: [PATCH 01/12] WIP --- paddle/framework/grad_op_builder.cc | 12 +- paddle/framework/op_registry.h | 178 +++++++++++----------------- paddle/framework/pybind.cc | 10 +- 3 files changed, 85 insertions(+), 115 deletions(-) diff --git a/paddle/framework/grad_op_builder.cc b/paddle/framework/grad_op_builder.cc index 6d032fb78f..ff8a5583af 100644 --- a/paddle/framework/grad_op_builder.cc +++ b/paddle/framework/grad_op_builder.cc @@ -76,8 +76,16 @@ static void TransOpArg(const OperatorBase* src_op, OperatorBase* dst_op, } OperatorBase* BuildGradOp(const OperatorBase* op) { - std::string grad_op_type = OpRegistry::grad_ops().at(op->type_); - OperatorBase* grad_op = OpRegistry::op_creators().at(grad_op_type)(); + auto it = op_info_map().find(op->type_); + PADDLE_ENFORCE(it != OpRegistry::op_info_map().end(), + "'%s' has not been registered.", op->type); + std::string grad_op_type = it->second.grad_op_type_; + PADDLE_ENFORCE(!grad_op_type.empty(), "'%s' has no gradient operator.", + op->type); + it = op_info_map().find(grad_op_type); + PADDLE_ENFORCE(it != OpRegistry::op_info_map().end(), + "'%s' has not been registered.", grad_op_type); + OperatorBase* grad_op = it->second.creator_(); grad_op->type_ = grad_op_type; grad_op->attrs_ = op->attrs_; grad_op->attrs_.erase("input_format"); diff --git a/paddle/framework/op_registry.h b/paddle/framework/op_registry.h index 84bf325fed..b88559f82b 100644 --- a/paddle/framework/op_registry.h +++ b/paddle/framework/op_registry.h @@ -17,6 +17,7 @@ limitations under the License. */ #include #include #include +#include #include #include #include "paddle/framework/attribute.h" @@ -174,6 +175,15 @@ Add a mark to which output is temporary is helpful for future optimization. bool has_temporary_output_{false}; }; +class NOPMaker : public OpProtoAndCheckerMaker {}; + +struct OpInfo { + std::function creator_; + std::string grad_op_type_; + OpProto* proto_; + OpAttrChecker* checker_; +}; + class OpRegistry { using OpCreator = std::function; using VarIndexMap = std::unordered_map; @@ -181,52 +191,55 @@ class OpRegistry { public: template - static void RegisterOp(const std::string& op_type) { - op_creators()[op_type] = [] { return new OpType; }; - OpAttrChecker& op_checker = op_checkers()[op_type]; - OpProto& op_proto = protos()[op_type]; - auto maker = ProtoMakerType(&op_proto, &op_checker); - maker.Validate(); - *op_proto.mutable_type() = op_type; - PADDLE_ENFORCE( - op_proto.IsInitialized(), - "Fail to initialize %s's OpProto, because %s is not initialized", - op_type, op_proto.InitializationErrorString()); - - VarIndexMaps()[op_type].reset(new VarIndexMap()); - auto& varmap = *VarIndexMaps()[op_type]; - int idx = 0; - for (auto& var : op_proto.inputs()) { - varmap[var.name()] = idx++; - } - idx = 0; - for (auto& var : op_proto.outputs()) { - varmap[var.name()] = idx++; + static void RegisterOp(const std::string& op_type, + const std::string& grad_op_type) { + PADDLE_ENFORCE(op_info_map().count(op_type) == 0, + "'%s' is registered more than once.", op_type); + OpInfo op_info; + op_info.creator_ = [] { return new OpType; }; + op_info.grad_op_type_ = grad_op_type; + if (std::type_index(typeid(ProtoMakerType)) != + std::type_index(typeid(NOPMaker))) { + op_info.proto_ = new OpProto; + op_info.op_checker_ = new OpAttrChecker; + auto maker = ProtoMakerType(op_info.proto_, op_info.op_checker_); + maker.Validate(); + *op_info.proto_->mutable_type() = op_type; + PADDLE_ENFORCE( + op_info.proto_->IsInitialized(), + "Fail to initialize %s's OpProto, because %s is not initialized", + op_type, op_info.proto_->InitializationErrorString()); + //======will be refactored in following PRs============// + VarIndexMaps()[op_type].reset(new VarIndexMap()); + auto& varmap = *VarIndexMaps()[op_type]; + int idx = 0; + for (auto& var : op_proto.inputs()) { + varmap[var.name()] = idx++; + } + idx = 0; + for (auto& var : op_proto.outputs()) { + varmap[var.name()] = idx++; + } + //================================================// } - } - - template - static void RegisterGradOp(const std::string& op_type, - const std::string& grad_op_type) { - op_creators()[grad_op_type] = [] { return new GradOpType; }; - grad_ops()[op_type] = grad_op_type; + op_info_map.insert(std::make_pair(op_type, op_info)); } static std::shared_ptr CreateOp(const std::string& type, const VarNameList& inputs, const VarNameList& outputs, const AttributeMap& attrs) { - auto op_create_it = op_creators().find(type); - PADDLE_ENFORCE(op_create_it != op_creators().end(), - "Operator %s cannot be found.", type); + auto it = op_info_map().find(type); + PADDLE_ENFORCE(it != op_info_map().end(), "'%s' has not been registered.", + type); - auto op = op_create_it->second(); + auto op = it->second.creator_(); op->type_ = type; op->inputs_ = inputs; op->outputs_ = outputs; op->attrs_ = attrs; - op_checkers().at(type).Check(op->attrs_); + it->second.checker_->Check(op->attrs_); GenerateTempVariableName(op); @@ -268,14 +281,9 @@ class OpRegistry { return grad_op; } - static std::unordered_map& protos() { - static std::unordered_map protos_; - return protos_; - } - - static std::unordered_map& grad_ops() { - static std::unordered_map grad_ops_; - return grad_ops_; + static std::unordered_map& op_info_map() { + static std::unordered_map op_info_map_; + return op_info_map_; } static std::unordered_map>& @@ -284,17 +292,7 @@ class OpRegistry { return maps_; } - static std::unordered_map& op_creators() { - static std::unordered_map op_creators_; - return op_creators_; - } - private: - static std::unordered_map& op_checkers() { - static std::unordered_map op_checkers_; - return op_checkers_; - } - static void GenerateTempVariableName(OperatorBase* op) { static std::atomic gUniqId(0UL); for (auto& outname : op->outputs_) { @@ -323,16 +321,9 @@ class Registrar { template class OpRegistrar : public Registrar { public: - explicit OpRegistrar(const char* op_type) { - OpRegistry::RegisterOp(op_type); - } -}; - -template -class GradOpRegistrar : public Registrar { - public: - GradOpRegistrar(const char* op_type, const char* grad_op_type) { - OpRegistry::RegisterGradOp(op_type, grad_op_type); + OpRegistrar(const char* op_type) { OpRegistrar(op_type, ""); } + OpRegistrar(const char* op_type, const char* grad_op_type) { + OpRegistry::RegisterOp(op_type, grad_op_type); } }; @@ -358,30 +349,21 @@ class OpKernelRegistrar : public Registrar { /** * Macro to register Operator. */ -#define REGISTER_OP(op_type, op_class, op_maker_class) \ +#define REGISTER_OP(op_type, op_class, op_maker_class, grad_op_type) \ STATIC_ASSERT_GLOBAL_NAMESPACE( \ __reg_op__##op_type, "REGISTER_OP must be called in global namespace"); \ static ::paddle::framework::OpRegistrar \ - __op_registrar_##op_type##__(#op_type); \ + __op_registrar_##op_type##__(#op_type, #grad_op_type); \ int TouchOpRegistrar_##op_type() { \ __op_registrar_##op_type##__.Touch(); \ return 0; \ } -/** - * Macro to register Gradient Operator. - */ -#define REGISTER_GRADIENT_OP(op_type, grad_op_type, grad_op_class) \ - STATIC_ASSERT_GLOBAL_NAMESPACE( \ - __reg_gradient_op__##op_type##_##grad_op_type, \ - "REGISTER_GRADIENT_OP must be called in global namespace"); \ - static ::paddle::framework::GradOpRegistrar \ - __op_gradient_registrar_##op_type##_##grad_op_type##__(#op_type, \ - #grad_op_type); \ - int TouchOpGradientRegistrar_##op_type() { \ - __op_gradient_registrar_##op_type##_##grad_op_type##__.Touch(); \ - return 0; \ - } +#define REGISTER_OP_WITHOUT_GRADIENT(op_type, op_class, op_maker_class) \ + REGISTER_OP(op_type, op_class, op_maker_class, ) + +#define REGISTER_GRADIENT_OP(op_type, op_class) \ + REGISTER_OP(op_type, op_class, ::paddle::framework::NOPMaker, ) /** * Macro to register OperatorKernel. @@ -400,10 +382,12 @@ class OpKernelRegistrar : public Registrar { /** * Macro to Forbid user register Gradient Operator. */ +/* #define NO_GRADIENT(op_type) \ - STATIC_ASSERT_GLOBAL_NAMESPACE( \ - __reg_gradient_op__##op_type##_##op_type##_grad, \ - "NO_GRADIENT must be called in global namespace") + STATIC_ASSERT_GLOBAL_NAMESPACE( \ + __reg_gradient_op__##op_type##_##op_type##_grad, \ + "NO_GRADIENT must be called in global namespace") +*/ #define REGISTER_OP_GPU_KERNEL(op_type, ...) \ REGISTER_OP_KERNEL(op_type, GPU, ::paddle::platform::GPUPlace, __VA_ARGS__) @@ -423,23 +407,6 @@ class OpKernelRegistrar : public Registrar { static int use_op_itself_##op_type##_ __attribute__((unused)) = \ TouchOpRegistrar_##op_type() -// TODO(fengjiayi): Most ops' gradient op have not been compeleted. So we use -// `NO_GRAD` to disable micro USE_OP_GRADIENT(op_type). Otherwise the code can't -// be compiled. `NO_GRAD` should be removed after all gradient ops are -// compeleted. -#define NO_GRAD -#ifndef NO_GRAD -#define USE_OP_GRADIENT(op_type) \ - STATIC_ASSERT_GLOBAL_NAMESPACE( \ - __use_op_gradient_##op_type, \ - "USE_OP_GRADIENT must be called in global namespace"); \ - extern int TouchOpGradientRegistrar_##op_type(); \ - static int use_op_gradient_##op_type##_ __attribute__((unused)) = \ - TouchOpGradientRegistrar_##op_type() -#else -#define USE_OP_GRADIENT(op_type) -#endif - #define USE_OP_DEVICE_KERNEL(op_type, DEVICE_TYPE) \ STATIC_ASSERT_GLOBAL_NAMESPACE( \ __use_op_kernel_##op_type##_##DEVICE_TYPE##__, \ @@ -459,18 +426,13 @@ class OpKernelRegistrar : public Registrar { USE_OP_DEVICE_KERNEL(op_type, GPU) #endif -#define USE_NO_GRAD_OP(op_type) \ - USE_OP_ITSELF(op_type); \ - USE_OP_KERNEL(op_type) +#define USE_CPU_ONLY_OP(op_type) \ + USE_OP_ITSELF(op_type); \ + USE_OP_DEVICE_KERNEL(op_type, CPU); -#define USE_CPU_OP(op_type) \ - USE_OP_ITSELF(op_type); \ - USE_OP_DEVICE_KERNEL(op_type, CPU); \ - USE_OP_GRADIENT(op_type) - -#define USE_OP(op_type) \ - USE_NO_GRAD_OP(op_type); \ - USE_OP_GRADIENT(op_type) +#define USE_OP(op_type) \ + USE_OP_ITSELF(op_type); \ + USE_OP_KERNEL(op_type) } // namespace framework } // namespace paddle diff --git a/paddle/framework/pybind.cc b/paddle/framework/pybind.cc index c18d38d2f9..412b416266 100644 --- a/paddle/framework/pybind.cc +++ b/paddle/framework/pybind.cc @@ -173,13 +173,13 @@ All parameter, weight, gradient are variables in Paddle. //! @note: Be careful! PyBind will return std::string as an unicode, not //! Python str. If you want a str object, you should cast them in Python. m.def("get_all_op_protos", []() -> std::vector { - auto &protos = OpRegistry::protos(); + auto &op_info_map = OpRegistry::op_info_map(); std::vector ret_values; - for (auto it = protos.begin(); it != protos.end(); ++it) { - PADDLE_ENFORCE(it->second.IsInitialized(), - "OpProto must all be initialized"); + for (auto it = op_info_map.begin(); it != op_info_map.end(); ++it) { + const OpProto *proto = it->second.proto_; + PADDLE_ENFORCE(proto->IsInitialized(), "OpProto must all be initialized"); std::string str; - PADDLE_ENFORCE(it->second.SerializeToString(&str), + PADDLE_ENFORCE(proto->SerializeToString(&str), "Serialize OpProto Error. This could be a bug of Paddle."); ret_values.push_back(py::bytes(str)); } From f784741d4aad7d57417fc60d9f956320c4779a9f Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Fri, 11 Aug 2017 11:59:22 -0700 Subject: [PATCH 02/12] Refine macro --- paddle/framework/backward_test.cc | 27 ++++++++++++------------ paddle/framework/grad_op_builder_test.cc | 8 +++---- paddle/framework/op_registry_test.cc | 8 +++---- paddle/framework/operator_test.cc | 10 +++++---- paddle/framework/pybind.cc | 4 ++-- paddle/operators/add_op.cc | 4 ++-- paddle/operators/cross_entropy_op.cc | 4 ++-- paddle/operators/fc_op.cc | 3 ++- paddle/operators/fill_zeros_like_op.cc | 3 ++- paddle/operators/gaussian_random_op.cc | 3 ++- paddle/operators/mean_op.cc | 4 ++-- paddle/operators/mul_op.cc | 4 ++-- paddle/operators/recurrent_op.cc | 5 +++-- paddle/operators/rowwise_add_op.cc | 3 ++- paddle/operators/sgd_op.cc | 2 +- paddle/operators/sigmoid_op.cc | 4 ++-- paddle/operators/softmax_op.cc | 4 ++-- paddle/operators/uniform_random_op.cc | 4 ++-- 18 files changed, 56 insertions(+), 48 deletions(-) diff --git a/paddle/framework/backward_test.cc b/paddle/framework/backward_test.cc index 1677a3ed4c..38194b716d 100644 --- a/paddle/framework/backward_test.cc +++ b/paddle/framework/backward_test.cc @@ -150,19 +150,20 @@ class AddOpMaker : public OpProtoAndCheckerMaker { namespace f = paddle::framework; namespace ops = paddle::operators; using EnforceNotMet = paddle::platform::EnforceNotMet; -REGISTER_OP(rowwise_add, f::EmptyOp, f::RowWiseAddOpMaker); -REGISTER_GRADIENT_OP(rowwise_add, rowwise_add_grad, f::EmptyOp); -REGISTER_OP(mul, f::EmptyOp, f::MulOpMaker); -REGISTER_GRADIENT_OP(mul, mul_grad, f::EmptyOp); -REGISTER_OP(sigmoid, f::EmptyOp, f::SigmoidOpMaker); -REGISTER_GRADIENT_OP(sigmoid, sigmoid_grad, f::EmptyOp); -REGISTER_OP(nograd, f::EmptyOp, f::NoGradOpMaker); -REGISTER_OP(fill_zeros_like, f::EmptyOp, f::FillZeroOpMaker); -REGISTER_OP(add, f::EmptyOp, f::AddOpMaker); -REGISTER_GRADIENT_OP(add, add_grad, f::EmptyOp); -REGISTER_OP(fc, f::FcOp, f::FcOpMaker); -REGISTER_OP(many_output_op, f::EmptyOp, f::ManyOutputOpMaker); -REGISTER_GRADIENT_OP(many_output_op, many_output_op_grad, f::EmptyOp); +REGISTER_OP(rowwise_add, f::EmptyOp, f::RowWiseAddOpMaker, rowwise_add_grad); +REGISTER_GRADIENT_OP(rowwise_add_grad, f::EmptyOp); +REGISTER_OP(mul, f::EmptyOp, f::MulOpMaker, mul_grad); +REGISTER_GRADIENT_OP(mul_grad, f::EmptyOp); +REGISTER_OP(sigmoid, f::EmptyOp, f::SigmoidOpMaker, sigmoid_grad); +REGISTER_GRADIENT_OP(sigmoid_grad, f::EmptyOp); +REGISTER_OP_WITHOUT_GRADIENT(nograd, f::EmptyOp, f::NoGradOpMaker); +REGISTER_OP_WITHOUT_GRADIENT(fill_zeros_like, f::EmptyOp, f::FillZeroOpMaker); +REGISTER_OP(add, f::EmptyOp, f::AddOpMaker, add_grad); +REGISTER_GRADIENT_OP(add_grad, f::EmptyOp); +REGISTER_OP_WITHOUT_GRADIENT(fc, f::FcOp, f::FcOpMaker); +REGISTER_OP(many_output_op, f::EmptyOp, f::ManyOutputOpMaker, + many_output_op_grad); +REGISTER_GRADIENT_OP(many_output_op_grad, f::EmptyOp); TEST(Backward, simple_op_grad) { auto fwd = f::OpRegistry::CreateOp("rowwise_add", {"X", "b"}, {"Out"}, {}); diff --git a/paddle/framework/grad_op_builder_test.cc b/paddle/framework/grad_op_builder_test.cc index f1ebbae52f..ad61b482e0 100644 --- a/paddle/framework/grad_op_builder_test.cc +++ b/paddle/framework/grad_op_builder_test.cc @@ -61,10 +61,10 @@ TEST(GradOpBuilder, AddTwo) { EXPECT_EQ(grad_add_op->Output("Y@GRAD"), "y@GRAD"); } -REGISTER_OP(mult_io, f::NOP, f::MutiInOutOpMaker); -REGISTER_GRADIENT_OP(mult_io, mult_io_grad, f::NOP); -REGISTER_OP(io_ignored, f::NOP, f::IOIgnoredOpMaker); -REGISTER_GRADIENT_OP(io_ignored, io_ignored_grad, f::NOP); +REGISTER_OP(mult_io, f::NOP, f::MutiInOutOpMaker, mult_io_grad); +REGISTER_GRADIENT_OP(mult_io_grad, f::NOP); +REGISTER_OP(io_ignored, f::NOP, f::IOIgnoredOpMaker, io_ignored_grad); +REGISTER_GRADIENT_OP(io_ignored_grad, f::NOP); TEST(GradOpBuilder, MutiInOut) { f::AttributeMap attrs{{"input_format", std::vector{0, 1, 4, 5}}, diff --git a/paddle/framework/op_registry_test.cc b/paddle/framework/op_registry_test.cc index 9894928a7a..6f21ffc8a4 100644 --- a/paddle/framework/op_registry_test.cc +++ b/paddle/framework/op_registry_test.cc @@ -49,10 +49,10 @@ class MyTestOpProtoAndCheckerMaker : public OpProtoAndCheckerMaker { } // namespace framework } // namespace paddle -REGISTER_OP(cos_sim, paddle::framework::CosineOp, - paddle::framework::CosineOpProtoAndCheckerMaker); -REGISTER_OP(my_test_op, paddle::framework::MyTestOp, - paddle::framework::MyTestOpProtoAndCheckerMaker); +REGISTER_OP_WITHOUT_GRADIENT(cos_sim, paddle::framework::CosineOp, + paddle::framework::CosineOpProtoAndCheckerMaker); +REGISTER_OP_WITHOUT_GRADIENT(my_test_op, paddle::framework::MyTestOp, + paddle::framework::MyTestOpProtoAndCheckerMaker); TEST(OpRegistry, CreateOp) { paddle::framework::OpDesc op_desc; diff --git a/paddle/framework/operator_test.cc b/paddle/framework/operator_test.cc index 387aada749..b1976a6514 100644 --- a/paddle/framework/operator_test.cc +++ b/paddle/framework/operator_test.cc @@ -54,8 +54,9 @@ class OpeWithoutKernelTestProtoAndCheckerMaker : public OpProtoAndCheckerMaker { } // namespace framework } // namespace paddle -REGISTER_OP(test_operator, paddle::framework::OpWithoutKernelTest, - paddle::framework::OpeWithoutKernelTestProtoAndCheckerMaker); +REGISTER_OP_WITHOUT_GRADIENT( + test_operator, paddle::framework::OpWithoutKernelTest, + paddle::framework::OpeWithoutKernelTestProtoAndCheckerMaker); TEST(OperatorBase, all) { paddle::framework::OpDesc op_desc; @@ -212,8 +213,9 @@ TEST(OpKernel, all) { ASSERT_EQ(paddle::framework::cpu_kernel_run_num, 1); } -REGISTER_OP(op_multi_inputs_with_kernel, paddle::framework::OpWithKernelTest, - paddle::framework::OpKernelTestMultiInputsProtoAndCheckerMaker); +REGISTER_OP_WITHOUT_GRADIENT( + op_multi_inputs_with_kernel, paddle::framework::OpWithKernelTest, + paddle::framework::OpKernelTestMultiInputsProtoAndCheckerMaker); REGISTER_OP_CPU_KERNEL(op_multi_inputs_with_kernel, paddle::framework::CPUKernalMultiInputsTest); diff --git a/paddle/framework/pybind.cc b/paddle/framework/pybind.cc index 412b416266..0416793d3a 100644 --- a/paddle/framework/pybind.cc +++ b/paddle/framework/pybind.cc @@ -30,9 +30,9 @@ limitations under the License. */ namespace py = pybind11; USE_OP(add_two); -USE_CPU_OP(onehot_cross_entropy); +USE_CPU_ONLY_OP(onehot_cross_entropy); USE_OP_ITSELF(fc); -USE_NO_GRAD_OP(sgd); +USE_OP(sgd); USE_OP(mul); USE_OP(mean); USE_OP(sigmoid); diff --git a/paddle/operators/add_op.cc b/paddle/operators/add_op.cc index 086245ef62..e8e26cbe9b 100644 --- a/paddle/operators/add_op.cc +++ b/paddle/operators/add_op.cc @@ -55,8 +55,8 @@ class AddOpGrad : public framework::OperatorWithKernel { } // namespace paddle namespace ops = paddle::operators; -REGISTER_OP(add_two, ops::AddOp, ops::AddOpMaker); -REGISTER_GRADIENT_OP(add_two, add_two_grad, ops::AddOpGrad); +REGISTER_OP(add_two, ops::AddOp, ops::AddOpMaker, add_two_grad); +REGISTER_GRADIENT_OP(add_two_grad, ops::AddOpGrad); REGISTER_OP_CPU_KERNEL(add_two, ops::AddKernel); diff --git a/paddle/operators/cross_entropy_op.cc b/paddle/operators/cross_entropy_op.cc index c813d54e17..7d0e74e5e4 100644 --- a/paddle/operators/cross_entropy_op.cc +++ b/paddle/operators/cross_entropy_op.cc @@ -69,11 +69,11 @@ OnehotCrossEntropy Operator. namespace ops = paddle::operators; REGISTER_OP(onehot_cross_entropy, ops::OnehotCrossEntropyOp, - ops::OnehotCrossEntropyOpMaker); + ops::OnehotCrossEntropyOpMaker, onehot_cross_entropy_grad); REGISTER_OP_CPU_KERNEL( onehot_cross_entropy, ops::OnehotCrossEntropyOpKernel); -REGISTER_GRADIENT_OP(onehot_cross_entropy, onehot_cross_entropy_grad, +REGISTER_GRADIENT_OP(onehot_cross_entropy_grad, ops::OnehotCrossEntropyGradientOp); REGISTER_OP_CPU_KERNEL( onehot_cross_entropy_grad, diff --git a/paddle/operators/fc_op.cc b/paddle/operators/fc_op.cc index 01a1a81206..9d32f327bf 100644 --- a/paddle/operators/fc_op.cc +++ b/paddle/operators/fc_op.cc @@ -73,4 +73,5 @@ USE_OP(sigmoid); USE_OP(softmax); namespace ops = paddle::operators; -REGISTER_OP(fc, ops::FullyConnectedOp, ops::FullyConnectedOpMaker); +REGISTER_OP_WITHOUT_GRADIENT(fc, ops::FullyConnectedOp, + ops::FullyConnectedOpMaker); diff --git a/paddle/operators/fill_zeros_like_op.cc b/paddle/operators/fill_zeros_like_op.cc index 3759a88678..d6fd368b07 100644 --- a/paddle/operators/fill_zeros_like_op.cc +++ b/paddle/operators/fill_zeros_like_op.cc @@ -51,7 +51,8 @@ The output will have the same size with input. } // namespace paddle namespace ops = paddle::operators; -REGISTER_OP(fill_zeros_like, ops::FillZerosLikeOp, ops::FillZerosLikeOpMaker); +REGISTER_OP_WITHOUT_GRADIENT(fill_zeros_like, ops::FillZerosLikeOp, + ops::FillZerosLikeOpMaker); REGISTER_OP_CPU_KERNEL( fill_zeros_like, ops::FillZerosLikeKernel); diff --git a/paddle/operators/gaussian_random_op.cc b/paddle/operators/gaussian_random_op.cc index ef417ae2f0..0bbbeaa08a 100644 --- a/paddle/operators/gaussian_random_op.cc +++ b/paddle/operators/gaussian_random_op.cc @@ -78,5 +78,6 @@ Use to initialize tensor with gaussian random generator. } // namespace paddle namespace ops = paddle::operators; -REGISTER_OP(gaussian_random, ops::GaussianRandomOp, ops::GaussianRandomOpMaker); +REGISTER_OP_WITHOUT_GRADIENT(gaussian_random, ops::GaussianRandomOp, + ops::GaussianRandomOpMaker); REGISTER_OP_CPU_KERNEL(gaussian_random, ops::GaussianRandomKernel); diff --git a/paddle/operators/mean_op.cc b/paddle/operators/mean_op.cc index 2ea049cb36..15e0708c46 100644 --- a/paddle/operators/mean_op.cc +++ b/paddle/operators/mean_op.cc @@ -50,9 +50,9 @@ class MeanGradOp : public framework::OperatorWithKernel { } // namespace paddle namespace ops = paddle::operators; -REGISTER_OP(mean, ops::MeanOp, ops::MeanOpMaker); +REGISTER_OP(mean, ops::MeanOp, ops::MeanOpMaker, mean_grad); REGISTER_OP_CPU_KERNEL(mean, ops::MeanKernel); -REGISTER_GRADIENT_OP(mean, mean_grad, ops::MeanGradOp); +REGISTER_GRADIENT_OP(mean_grad, ops::MeanGradOp); REGISTER_OP_CPU_KERNEL(mean_grad, ops::MeanGradKernel); diff --git a/paddle/operators/mul_op.cc b/paddle/operators/mul_op.cc index db81fd555d..60550a2742 100644 --- a/paddle/operators/mul_op.cc +++ b/paddle/operators/mul_op.cc @@ -65,7 +65,7 @@ class MulOpGrad : public framework::OperatorWithKernel { } // namespace paddle namespace ops = paddle::operators; -REGISTER_OP(mul, ops::MulOp, ops::MulOpMaker); -REGISTER_GRADIENT_OP(mul, mul_grad, ops::MulOpGrad); +REGISTER_OP(mul, ops::MulOp, ops::MulOpMaker, mul_grad); +REGISTER_GRADIENT_OP(mul_grad, ops::MulOpGrad); REGISTER_OP_CPU_KERNEL(mul, ops::MulKernel); diff --git a/paddle/operators/recurrent_op.cc b/paddle/operators/recurrent_op.cc index 2438374205..91be1ce519 100644 --- a/paddle/operators/recurrent_op.cc +++ b/paddle/operators/recurrent_op.cc @@ -235,5 +235,6 @@ void RecurrentGradientOp::Init() { } // namespace operators } // namespace paddle -REGISTER_OP(recurrent_op, paddle::operators::RecurrentOp, - paddle::operators::RecurrentAlgorithmProtoAndCheckerMaker); +REGISTER_OP_WITHOUT_GRADIENT( + recurrent_op, paddle::operators::RecurrentOp, + paddle::operators::RecurrentAlgorithmProtoAndCheckerMaker); diff --git a/paddle/operators/rowwise_add_op.cc b/paddle/operators/rowwise_add_op.cc index 55ed1c2f4c..262a4127ef 100644 --- a/paddle/operators/rowwise_add_op.cc +++ b/paddle/operators/rowwise_add_op.cc @@ -53,6 +53,7 @@ for i in xrange(X.shape[0]): } // namespace paddle namespace ops = paddle::operators; -REGISTER_OP(rowwise_add, ops::RowWiseAddOp, ops::RowWiseAddOpMaker); +REGISTER_OP_WITHOUT_GRADIENT(rowwise_add, ops::RowWiseAddOp, + ops::RowWiseAddOpMaker); REGISTER_OP_CPU_KERNEL( rowwise_add, ops::RowWiseAddKernel); diff --git a/paddle/operators/sgd_op.cc b/paddle/operators/sgd_op.cc index f9a28ff8a6..94d0fe0466 100644 --- a/paddle/operators/sgd_op.cc +++ b/paddle/operators/sgd_op.cc @@ -52,6 +52,6 @@ param_out = param - learning_rate * grad; } // namespace paddle namespace ops = paddle::operators; -REGISTER_OP(sgd, ops::SGDOp, ops::SGDOpMaker); +REGISTER_OP_WITHOUT_GRADIENT(sgd, ops::SGDOp, ops::SGDOpMaker); REGISTER_OP_CPU_KERNEL(sgd, ops::SGDOpKernel); diff --git a/paddle/operators/sigmoid_op.cc b/paddle/operators/sigmoid_op.cc index bc5e0bbb18..fb27ffbfa1 100644 --- a/paddle/operators/sigmoid_op.cc +++ b/paddle/operators/sigmoid_op.cc @@ -48,8 +48,8 @@ class SigmoidOpGrad : public framework::OperatorWithKernel { } // namespace paddle namespace ops = paddle::operators; -REGISTER_OP(sigmoid, ops::SigmoidOp, ops::SigmoidOpMaker); -REGISTER_GRADIENT_OP(sigmoid, sigmoid_grad, ops::SigmoidOpGrad); +REGISTER_OP(sigmoid, ops::SigmoidOp, ops::SigmoidOpMaker, sigmoid_grad); +REGISTER_GRADIENT_OP(sigmoid_grad, ops::SigmoidOpGrad); REGISTER_OP_CPU_KERNEL(sigmoid, ops::SigmoidKernel); diff --git a/paddle/operators/softmax_op.cc b/paddle/operators/softmax_op.cc index 3dd4e86918..abc21337c5 100644 --- a/paddle/operators/softmax_op.cc +++ b/paddle/operators/softmax_op.cc @@ -64,9 +64,9 @@ class SoftmaxOpGrad : public framework::OperatorWithKernel { namespace ops = paddle::operators; -REGISTER_OP(softmax, ops::SoftmaxOp, ops::SoftmaxOpMaker); +REGISTER_OP(softmax, ops::SoftmaxOp, ops::SoftmaxOpMaker, softmax_grad); REGISTER_OP_CPU_KERNEL(softmax, ops::SoftmaxKernel); -REGISTER_GRADIENT_OP(softmax, softmax_grad, ops::SoftmaxOpGrad); +REGISTER_GRADIENT_OP(softmax_grad, ops::SoftmaxOpGrad); REGISTER_OP_CPU_KERNEL( softmax_grad, ops::SoftmaxGradKernel); diff --git a/paddle/operators/uniform_random_op.cc b/paddle/operators/uniform_random_op.cc index 405b84b76d..37ec7fe427 100644 --- a/paddle/operators/uniform_random_op.cc +++ b/paddle/operators/uniform_random_op.cc @@ -78,7 +78,7 @@ Used to initialize tensor with uniform random generator. } // namespace operators } // namespace paddle -REGISTER_OP(uniform_random, paddle::operators::UniformRandomOp, - paddle::operators::UniformRandomOpMaker); +REGISTER_OP_WITHOUT_GRADIENT(uniform_random, paddle::operators::UniformRandomOp, + paddle::operators::UniformRandomOpMaker); REGISTER_OP_CPU_KERNEL(uniform_random, paddle::operators::CPUUniformRandomKernel); From 6768b31037161fa8a9979bd2b4294adbf11966c2 Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Fri, 11 Aug 2017 13:43:31 -0700 Subject: [PATCH 03/12] Fix compile error --- paddle/framework/grad_op_builder.cc | 10 +++++----- paddle/framework/op_registry.h | 29 ++++++++++++++++------------- paddle/framework/operator_test.cc | 5 +++-- 3 files changed, 24 insertions(+), 20 deletions(-) diff --git a/paddle/framework/grad_op_builder.cc b/paddle/framework/grad_op_builder.cc index ff8a5583af..f534b2c336 100644 --- a/paddle/framework/grad_op_builder.cc +++ b/paddle/framework/grad_op_builder.cc @@ -50,7 +50,7 @@ static void TransOpArg(const OperatorBase* src_op, OperatorBase* dst_op, std::vector& dst_inout = dst_type == OpArgType::IN ? dst_op->inputs_ : dst_op->outputs_; std::vector* dst_format = GetOpFormat(dst_op, dst_type); - const OpProto& proto = OpRegistry::protos().at(src_op->type_); + const OpProto& proto = *(OpRegistry::op_info_map().at(src_op->type_).proto_); const auto& src_arg_list = src_type == OpArgType::IN ? proto.inputs() : proto.outputs(); @@ -76,13 +76,13 @@ static void TransOpArg(const OperatorBase* src_op, OperatorBase* dst_op, } OperatorBase* BuildGradOp(const OperatorBase* op) { - auto it = op_info_map().find(op->type_); + auto it = OpRegistry::op_info_map().find(op->type_); PADDLE_ENFORCE(it != OpRegistry::op_info_map().end(), - "'%s' has not been registered.", op->type); + "'%s' has not been registered.", op->type_); std::string grad_op_type = it->second.grad_op_type_; PADDLE_ENFORCE(!grad_op_type.empty(), "'%s' has no gradient operator.", - op->type); - it = op_info_map().find(grad_op_type); + op->type_); + it = OpRegistry::op_info_map().find(grad_op_type); PADDLE_ENFORCE(it != OpRegistry::op_info_map().end(), "'%s' has not been registered.", grad_op_type); OperatorBase* grad_op = it->second.creator_(); diff --git a/paddle/framework/op_registry.h b/paddle/framework/op_registry.h index b88559f82b..69c5f549e3 100644 --- a/paddle/framework/op_registry.h +++ b/paddle/framework/op_registry.h @@ -175,17 +175,20 @@ Add a mark to which output is temporary is helpful for future optimization. bool has_temporary_output_{false}; }; -class NOPMaker : public OpProtoAndCheckerMaker {}; +class NOPMaker : public OpProtoAndCheckerMaker { + public: + NOPMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) {} +}; struct OpInfo { - std::function creator_; + std::function creator_; std::string grad_op_type_; OpProto* proto_; OpAttrChecker* checker_; }; class OpRegistry { - using OpCreator = std::function; using VarIndexMap = std::unordered_map; using VarNameList = std::vector; @@ -201,28 +204,28 @@ class OpRegistry { if (std::type_index(typeid(ProtoMakerType)) != std::type_index(typeid(NOPMaker))) { op_info.proto_ = new OpProto; - op_info.op_checker_ = new OpAttrChecker; - auto maker = ProtoMakerType(op_info.proto_, op_info.op_checker_); + op_info.checker_ = new OpAttrChecker; + auto maker = ProtoMakerType(op_info.proto_, op_info.checker_); maker.Validate(); *op_info.proto_->mutable_type() = op_type; PADDLE_ENFORCE( op_info.proto_->IsInitialized(), "Fail to initialize %s's OpProto, because %s is not initialized", op_type, op_info.proto_->InitializationErrorString()); - //======will be refactored in following PRs============// + // ======will be refactored in following PRs============ // VarIndexMaps()[op_type].reset(new VarIndexMap()); auto& varmap = *VarIndexMaps()[op_type]; int idx = 0; - for (auto& var : op_proto.inputs()) { + for (auto& var : op_info.proto_->inputs()) { varmap[var.name()] = idx++; } idx = 0; - for (auto& var : op_proto.outputs()) { + for (auto& var : op_info.proto_->outputs()) { varmap[var.name()] = idx++; } - //================================================// + // ================================================ // } - op_info_map.insert(std::make_pair(op_type, op_info)); + op_info_map().insert(std::make_pair(op_type, op_info)); } static std::shared_ptr CreateOp(const std::string& type, @@ -281,8 +284,8 @@ class OpRegistry { return grad_op; } - static std::unordered_map& op_info_map() { - static std::unordered_map op_info_map_; + static std::unordered_map& op_info_map() { + static std::unordered_map op_info_map_; return op_info_map_; } @@ -321,7 +324,7 @@ class Registrar { template class OpRegistrar : public Registrar { public: - OpRegistrar(const char* op_type) { OpRegistrar(op_type, ""); } + explicit OpRegistrar(const char* op_type) { OpRegistrar(op_type, ""); } OpRegistrar(const char* op_type, const char* grad_op_type) { OpRegistry::RegisterOp(op_type, grad_op_type); } diff --git a/paddle/framework/operator_test.cc b/paddle/framework/operator_test.cc index b1976a6514..3887cadc60 100644 --- a/paddle/framework/operator_test.cc +++ b/paddle/framework/operator_test.cc @@ -188,8 +188,9 @@ class CPUKernalMultiInputsTest : public OpKernel { } // namespace framework } // namespace paddle -REGISTER_OP(op_with_kernel, paddle::framework::OpWithKernelTest, - paddle::framework::OpKernelTestProtoAndCheckerMaker); +REGISTER_OP_WITHOUT_GRADIENT( + op_with_kernel, paddle::framework::OpWithKernelTest, + paddle::framework::OpKernelTestProtoAndCheckerMaker); REGISTER_OP_CPU_KERNEL(op_with_kernel, paddle::framework::CPUKernelTest); From 2ea2fbea1bfb6f73c87f7029953ba8007e8cf4fb Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Fri, 11 Aug 2017 17:30:49 -0700 Subject: [PATCH 04/12] Merge REGISTER_OP and REGISTER_GRADIENT_OP --- paddle/framework/backward_test.cc | 16 ++++++---------- paddle/framework/grad_op_builder_test.cc | 13 ++----------- paddle/framework/op_registry.h | 22 +++++++++++++--------- paddle/framework/operator.h | 7 +++++++ paddle/operators/add_op.cc | 3 +-- paddle/operators/cross_entropy_op.cc | 5 ++--- paddle/operators/mean_op.cc | 3 +-- paddle/operators/mul_op.cc | 4 +--- paddle/operators/sigmoid_op.cc | 5 ++--- paddle/operators/softmax_op.cc | 4 ++-- 10 files changed, 37 insertions(+), 45 deletions(-) diff --git a/paddle/framework/backward_test.cc b/paddle/framework/backward_test.cc index 38194b716d..4136e2c36a 100644 --- a/paddle/framework/backward_test.cc +++ b/paddle/framework/backward_test.cc @@ -150,20 +150,16 @@ class AddOpMaker : public OpProtoAndCheckerMaker { namespace f = paddle::framework; namespace ops = paddle::operators; using EnforceNotMet = paddle::platform::EnforceNotMet; -REGISTER_OP(rowwise_add, f::EmptyOp, f::RowWiseAddOpMaker, rowwise_add_grad); -REGISTER_GRADIENT_OP(rowwise_add_grad, f::EmptyOp); -REGISTER_OP(mul, f::EmptyOp, f::MulOpMaker, mul_grad); -REGISTER_GRADIENT_OP(mul_grad, f::EmptyOp); -REGISTER_OP(sigmoid, f::EmptyOp, f::SigmoidOpMaker, sigmoid_grad); -REGISTER_GRADIENT_OP(sigmoid_grad, f::EmptyOp); +REGISTER_OP(rowwise_add, f::EmptyOp, f::RowWiseAddOpMaker, rowwise_add_grad, + f::EmptyOp); +REGISTER_OP(mul, f::EmptyOp, f::MulOpMaker, mul_grad, f::EmptyOp); +REGISTER_OP(sigmoid, f::EmptyOp, f::SigmoidOpMaker, sigmoid_grad, f::EmptyOp); REGISTER_OP_WITHOUT_GRADIENT(nograd, f::EmptyOp, f::NoGradOpMaker); REGISTER_OP_WITHOUT_GRADIENT(fill_zeros_like, f::EmptyOp, f::FillZeroOpMaker); -REGISTER_OP(add, f::EmptyOp, f::AddOpMaker, add_grad); -REGISTER_GRADIENT_OP(add_grad, f::EmptyOp); +REGISTER_OP(add, f::EmptyOp, f::AddOpMaker, add_grad, f::EmptyOp); REGISTER_OP_WITHOUT_GRADIENT(fc, f::FcOp, f::FcOpMaker); REGISTER_OP(many_output_op, f::EmptyOp, f::ManyOutputOpMaker, - many_output_op_grad); -REGISTER_GRADIENT_OP(many_output_op_grad, f::EmptyOp); + many_output_op_grad, f::EmptyOp); TEST(Backward, simple_op_grad) { auto fwd = f::OpRegistry::CreateOp("rowwise_add", {"X", "b"}, {"Out"}, {}); diff --git a/paddle/framework/grad_op_builder_test.cc b/paddle/framework/grad_op_builder_test.cc index ad61b482e0..3d7f1a753d 100644 --- a/paddle/framework/grad_op_builder_test.cc +++ b/paddle/framework/grad_op_builder_test.cc @@ -8,13 +8,6 @@ USE_OP(add_two); namespace paddle { namespace framework { -class NOP : public OperatorBase { - public: - void InferShape(const Scope &scope) const override {} - void Run(const Scope &scope, - const platform::DeviceContext &dev_ctx) const override {} -}; - class MutiInOutOpMaker : public OpProtoAndCheckerMaker { public: MutiInOutOpMaker(OpProto *proto, OpAttrChecker *op_checker) @@ -61,10 +54,8 @@ TEST(GradOpBuilder, AddTwo) { EXPECT_EQ(grad_add_op->Output("Y@GRAD"), "y@GRAD"); } -REGISTER_OP(mult_io, f::NOP, f::MutiInOutOpMaker, mult_io_grad); -REGISTER_GRADIENT_OP(mult_io_grad, f::NOP); -REGISTER_OP(io_ignored, f::NOP, f::IOIgnoredOpMaker, io_ignored_grad); -REGISTER_GRADIENT_OP(io_ignored_grad, f::NOP); +REGISTER_OP(mult_io, f::NOP, f::MutiInOutOpMaker, mult_io_grad, f::NOP); +REGISTER_OP(io_ignored, f::NOP, f::IOIgnoredOpMaker, io_ignored_grad, f::NOP); TEST(GradOpBuilder, MutiInOut) { f::AttributeMap attrs{{"input_format", std::vector{0, 1, 4, 5}}, diff --git a/paddle/framework/op_registry.h b/paddle/framework/op_registry.h index 69c5f549e3..080a7149bb 100644 --- a/paddle/framework/op_registry.h +++ b/paddle/framework/op_registry.h @@ -193,7 +193,7 @@ class OpRegistry { using VarNameList = std::vector; public: - template + template static void RegisterOp(const std::string& op_type, const std::string& grad_op_type) { PADDLE_ENFORCE(op_info_map().count(op_type) == 0, @@ -226,6 +226,10 @@ class OpRegistry { // ================================================ // } op_info_map().insert(std::make_pair(op_type, op_info)); + // register gradient op + if (!grad_op_type.empty()) { + RegisterOp(grad_op_type, ""); + } } static std::shared_ptr CreateOp(const std::string& type, @@ -321,12 +325,13 @@ class Registrar { void Touch() {} }; -template +template class OpRegistrar : public Registrar { public: explicit OpRegistrar(const char* op_type) { OpRegistrar(op_type, ""); } OpRegistrar(const char* op_type, const char* grad_op_type) { - OpRegistry::RegisterOp(op_type, grad_op_type); + OpRegistry::RegisterOp(op_type, + grad_op_type); } }; @@ -352,10 +357,12 @@ class OpKernelRegistrar : public Registrar { /** * Macro to register Operator. */ -#define REGISTER_OP(op_type, op_class, op_maker_class, grad_op_type) \ +#define REGISTER_OP(op_type, op_class, op_maker_class, grad_op_type, \ + grad_op_class) \ STATIC_ASSERT_GLOBAL_NAMESPACE( \ __reg_op__##op_type, "REGISTER_OP must be called in global namespace"); \ - static ::paddle::framework::OpRegistrar \ + static ::paddle::framework::OpRegistrar \ __op_registrar_##op_type##__(#op_type, #grad_op_type); \ int TouchOpRegistrar_##op_type() { \ __op_registrar_##op_type##__.Touch(); \ @@ -363,10 +370,7 @@ class OpKernelRegistrar : public Registrar { } #define REGISTER_OP_WITHOUT_GRADIENT(op_type, op_class, op_maker_class) \ - REGISTER_OP(op_type, op_class, op_maker_class, ) - -#define REGISTER_GRADIENT_OP(op_type, op_class) \ - REGISTER_OP(op_type, op_class, ::paddle::framework::NOPMaker, ) + REGISTER_OP(op_type, op_class, op_maker_class, , ::paddle::framework::NOP) /** * Macro to register OperatorKernel. diff --git a/paddle/framework/operator.h b/paddle/framework/operator.h index f5d167a16e..13308e0dae 100644 --- a/paddle/framework/operator.h +++ b/paddle/framework/operator.h @@ -125,6 +125,13 @@ class OperatorBase { std::shared_ptr> in_out_idxs_; }; +class NOP : public OperatorBase { + public: + void InferShape(const Scope& scope) const override {} + void Run(const Scope& scope, + const platform::DeviceContext& dev_ctx) const override {} +}; + class InferShapeContext { public: InferShapeContext(const OperatorBase& op, const Scope& scope) diff --git a/paddle/operators/add_op.cc b/paddle/operators/add_op.cc index e8e26cbe9b..447e7b3915 100644 --- a/paddle/operators/add_op.cc +++ b/paddle/operators/add_op.cc @@ -55,8 +55,7 @@ class AddOpGrad : public framework::OperatorWithKernel { } // namespace paddle namespace ops = paddle::operators; -REGISTER_OP(add_two, ops::AddOp, ops::AddOpMaker, add_two_grad); -REGISTER_GRADIENT_OP(add_two_grad, ops::AddOpGrad); +REGISTER_OP(add_two, ops::AddOp, ops::AddOpMaker, add_two_grad, ops::AddOpGrad); REGISTER_OP_CPU_KERNEL(add_two, ops::AddKernel); diff --git a/paddle/operators/cross_entropy_op.cc b/paddle/operators/cross_entropy_op.cc index 7d0e74e5e4..3dcaccd756 100644 --- a/paddle/operators/cross_entropy_op.cc +++ b/paddle/operators/cross_entropy_op.cc @@ -69,12 +69,11 @@ OnehotCrossEntropy Operator. namespace ops = paddle::operators; REGISTER_OP(onehot_cross_entropy, ops::OnehotCrossEntropyOp, - ops::OnehotCrossEntropyOpMaker, onehot_cross_entropy_grad); + ops::OnehotCrossEntropyOpMaker, onehot_cross_entropy_grad, + ops::OnehotCrossEntropyGradientOp); REGISTER_OP_CPU_KERNEL( onehot_cross_entropy, ops::OnehotCrossEntropyOpKernel); -REGISTER_GRADIENT_OP(onehot_cross_entropy_grad, - ops::OnehotCrossEntropyGradientOp); REGISTER_OP_CPU_KERNEL( onehot_cross_entropy_grad, ops::OnehotCrossEntropyGradientOpKernel); diff --git a/paddle/operators/mean_op.cc b/paddle/operators/mean_op.cc index 15e0708c46..c41208014a 100644 --- a/paddle/operators/mean_op.cc +++ b/paddle/operators/mean_op.cc @@ -50,9 +50,8 @@ class MeanGradOp : public framework::OperatorWithKernel { } // namespace paddle namespace ops = paddle::operators; -REGISTER_OP(mean, ops::MeanOp, ops::MeanOpMaker, mean_grad); +REGISTER_OP(mean, ops::MeanOp, ops::MeanOpMaker, mean_grad, ops::MeanGradOp); REGISTER_OP_CPU_KERNEL(mean, ops::MeanKernel); -REGISTER_GRADIENT_OP(mean_grad, ops::MeanGradOp); REGISTER_OP_CPU_KERNEL(mean_grad, ops::MeanGradKernel); diff --git a/paddle/operators/mul_op.cc b/paddle/operators/mul_op.cc index 60550a2742..0c4547f04d 100644 --- a/paddle/operators/mul_op.cc +++ b/paddle/operators/mul_op.cc @@ -65,7 +65,5 @@ class MulOpGrad : public framework::OperatorWithKernel { } // namespace paddle namespace ops = paddle::operators; -REGISTER_OP(mul, ops::MulOp, ops::MulOpMaker, mul_grad); -REGISTER_GRADIENT_OP(mul_grad, ops::MulOpGrad); - +REGISTER_OP(mul, ops::MulOp, ops::MulOpMaker, mul_grad, ops::MulOpGrad); REGISTER_OP_CPU_KERNEL(mul, ops::MulKernel); diff --git a/paddle/operators/sigmoid_op.cc b/paddle/operators/sigmoid_op.cc index fb27ffbfa1..4f3a880b40 100644 --- a/paddle/operators/sigmoid_op.cc +++ b/paddle/operators/sigmoid_op.cc @@ -48,9 +48,8 @@ class SigmoidOpGrad : public framework::OperatorWithKernel { } // namespace paddle namespace ops = paddle::operators; -REGISTER_OP(sigmoid, ops::SigmoidOp, ops::SigmoidOpMaker, sigmoid_grad); -REGISTER_GRADIENT_OP(sigmoid_grad, ops::SigmoidOpGrad); - +REGISTER_OP(sigmoid, ops::SigmoidOp, ops::SigmoidOpMaker, sigmoid_grad, + ops::SigmoidOpGrad); REGISTER_OP_CPU_KERNEL(sigmoid, ops::SigmoidKernel); REGISTER_OP_CPU_KERNEL( diff --git a/paddle/operators/softmax_op.cc b/paddle/operators/softmax_op.cc index abc21337c5..99bc5b77d1 100644 --- a/paddle/operators/softmax_op.cc +++ b/paddle/operators/softmax_op.cc @@ -64,9 +64,9 @@ class SoftmaxOpGrad : public framework::OperatorWithKernel { namespace ops = paddle::operators; -REGISTER_OP(softmax, ops::SoftmaxOp, ops::SoftmaxOpMaker, softmax_grad); +REGISTER_OP(softmax, ops::SoftmaxOp, ops::SoftmaxOpMaker, softmax_grad, + ops::SoftmaxOpGrad); REGISTER_OP_CPU_KERNEL(softmax, ops::SoftmaxKernel); -REGISTER_GRADIENT_OP(softmax_grad, ops::SoftmaxOpGrad); REGISTER_OP_CPU_KERNEL( softmax_grad, ops::SoftmaxGradKernel); From e0ccc178bc3201d10c916eb2be2a6261c7472495 Mon Sep 17 00:00:00 2001 From: dong zhihong Date: Sat, 12 Aug 2017 15:50:41 +0800 Subject: [PATCH 05/12] simplify docker script --- Dockerfile | 17 +++++++++++------ python/requirements.txt | 9 +++++++++ 2 files changed, 20 insertions(+), 6 deletions(-) create mode 100644 python/requirements.txt diff --git a/Dockerfile b/Dockerfile index 41b6729124..3ef3aa14f4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,7 +35,7 @@ RUN apt-get update && \ apt-get clean -y # paddle is using numpy.flip, which is introduced since 1.12.0 -RUN pip --no-cache-dir install 'numpy>=1.12.0' +# RUN pip --no-cache-dir install 'numpy>=1.12.0' # Install Go and glide RUN wget -qO- https://storage.googleapis.com/golang/go1.8.1.linux-amd64.tar.gz | \ @@ -58,13 +58,18 @@ RUN localedef -i en_US -f UTF-8 en_US.UTF-8 # FIXME: due to temporary ipykernel dependency issue, specify ipykernel jupyter # version util jupyter fixes this issue. RUN pip install --upgrade pip && \ - pip install -U 'protobuf==3.1.0' && \ - pip install -U wheel pillow BeautifulSoup && \ + pip install -U wheel && \ pip install -U docopt PyYAML sphinx && \ - pip install -U sphinx-rtd-theme==0.1.9 recommonmark && \ - pip install pre-commit 'requests==2.9.2' 'ipython==5.3.0' && \ + pip install -U sphinx-rtd-theme==0.1.9 recommonmark + +RUN pip install pre-commit 'ipython==5.3.0' && \ pip install 'ipykernel==4.6.0' 'jupyter==1.0.0' && \ - pip install opencv-python rarfile 'scipy>=0.19.0' 'nltk>=3.2.2' + pip install opencv-python + +# paddle is using numpy.flip, which is introduced since 1.12.0 +RUN pip --no-cache-dir install 'numpy>=1.12.0' +COPY ./python/requirements.txt /root/ +RUN pip install -r /root/requirements.txt # To fix https://github.com/PaddlePaddle/Paddle/issues/1954, we use # the solution in https://urllib3.readthedocs.io/en/latest/user-guide.html#ssl-py2 diff --git a/python/requirements.txt b/python/requirements.txt new file mode 100644 index 0000000000..3df822bd76 --- /dev/null +++ b/python/requirements.txt @@ -0,0 +1,9 @@ +requests==2.9.2 +numpy>=1.12 +protobuf==3.1 +recordio +matplotlib +rarfile +scipy>=0.19.0 +Pillow +nltk>=3.2.2 From 19dfe1f38bcb100cc8e3e1b50ef891b32d614223 Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Sun, 13 Aug 2017 13:57:49 -0700 Subject: [PATCH 06/12] fix compile errors --- paddle/framework/operator.h | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/paddle/framework/operator.h b/paddle/framework/operator.h index 058fe3282f..c8c49b5455 100644 --- a/paddle/framework/operator.h +++ b/paddle/framework/operator.h @@ -55,6 +55,16 @@ class OperatorBase; class InferShapeContext; class ExecutionContext; +#define DEFINE_OPERATOR_CTOR(Class, ParentClass) \ + public: \ + Class() { /* TODO(yi): This constructor is to be removed. */ \ + } \ + Class(const std::string& type, const std::vector& inputs, \ + const std::vector& outputs, \ + const ::paddle::framework::AttributeMap& attrs, \ + std::unordered_map* in_out_idxs) \ + : ParentClass(type, inputs, outputs, attrs, in_out_idxs) {} + /** * OperatorBase has the basic element that Net will call to do computation. * Only CreateOperator from OpRegistry will new Operator directly. User @@ -366,15 +376,5 @@ class OperatorWithKernel : public OperatorBase { virtual void InferShape(const InferShapeContext& ctx) const = 0; }; -#define DEFINE_OPERATOR_CTOR(Class, ParentClass) \ - public: \ - Class() { /* TODO(yi): This constructor is to be removed. */ \ - } \ - Class(const std::string& type, const std::vector& inputs, \ - const std::vector& outputs, \ - const ::paddle::framework::AttributeMap& attrs, \ - std::unordered_map* in_out_idxs) \ - : ParentClass(type, inputs, outputs, attrs, in_out_idxs) {} - } // namespace framework } // namespace paddle From fb6bec6a8f5d8cb57773e2ca1e438476fa695892 Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Sun, 13 Aug 2017 15:19:40 -0700 Subject: [PATCH 07/12] Fix a bug --- paddle/framework/op_registry.h | 3 +++ paddle/framework/pybind.cc | 3 +++ 2 files changed, 6 insertions(+) diff --git a/paddle/framework/op_registry.h b/paddle/framework/op_registry.h index a561b5f48e..23f641cba2 100644 --- a/paddle/framework/op_registry.h +++ b/paddle/framework/op_registry.h @@ -222,6 +222,9 @@ class OpRegistry { varmap[var.name()] = idx++; } // ================================================ // + } else { + op_info.proto_ = nullptr; + op_info.checker_ = nullptr; } op_info_map().insert(std::make_pair(op_type, op_info)); // register gradient op diff --git a/paddle/framework/pybind.cc b/paddle/framework/pybind.cc index 3343a51c8d..56a89d87fd 100644 --- a/paddle/framework/pybind.cc +++ b/paddle/framework/pybind.cc @@ -176,6 +176,9 @@ All parameter, weight, gradient are variables in Paddle. std::vector ret_values; for (auto it = op_info_map.begin(); it != op_info_map.end(); ++it) { const OpProto *proto = it->second.proto_; + if (proto == nullptr) { + continue; + } PADDLE_ENFORCE(proto->IsInitialized(), "OpProto must all be initialized"); std::string str; PADDLE_ENFORCE(proto->SerializeToString(&str), From 0405e88440aac1788b1bb9eef3303fe301842f9c Mon Sep 17 00:00:00 2001 From: dong zhihong Date: Mon, 14 Aug 2017 14:43:14 +0800 Subject: [PATCH 08/12] fix numpy docker --- Dockerfile | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3ef3aa14f4..885bec9ba0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,9 +34,6 @@ RUN apt-get update && \ net-tools && \ apt-get clean -y -# paddle is using numpy.flip, which is introduced since 1.12.0 -# RUN pip --no-cache-dir install 'numpy>=1.12.0' - # Install Go and glide RUN wget -qO- https://storage.googleapis.com/golang/go1.8.1.linux-amd64.tar.gz | \ tar -xz -C /usr/local && \ @@ -67,7 +64,7 @@ RUN pip install pre-commit 'ipython==5.3.0' && \ pip install opencv-python # paddle is using numpy.flip, which is introduced since 1.12.0 -RUN pip --no-cache-dir install 'numpy>=1.12.0' +# RUN pip --no-cache-dir install 'numpy>=1.12.0' COPY ./python/requirements.txt /root/ RUN pip install -r /root/requirements.txt From e9e6abfa64cd7395c9bc0a934b0fe47853fb8437 Mon Sep 17 00:00:00 2001 From: dong zhihong Date: Mon, 14 Aug 2017 12:10:36 +0000 Subject: [PATCH 09/12] remove comments --- Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 885bec9ba0..da00471025 100644 --- a/Dockerfile +++ b/Dockerfile @@ -63,8 +63,6 @@ RUN pip install pre-commit 'ipython==5.3.0' && \ pip install 'ipykernel==4.6.0' 'jupyter==1.0.0' && \ pip install opencv-python -# paddle is using numpy.flip, which is introduced since 1.12.0 -# RUN pip --no-cache-dir install 'numpy>=1.12.0' COPY ./python/requirements.txt /root/ RUN pip install -r /root/requirements.txt From edb541f2926c6ef2cd7c9b1c5d0c80f692a50697 Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Mon, 14 Aug 2017 11:47:00 -0700 Subject: [PATCH 10/12] fix compile errors --- paddle/framework/grad_op_builder.cc | 5 ++++- paddle/framework/op_registry.h | 20 ++++++++++---------- paddle/framework/operator.cc | 2 +- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/paddle/framework/grad_op_builder.cc b/paddle/framework/grad_op_builder.cc index b316f2d535..cb491ec95f 100644 --- a/paddle/framework/grad_op_builder.cc +++ b/paddle/framework/grad_op_builder.cc @@ -25,8 +25,9 @@ static void TransOpArg(const OperatorBase* src_op, const OpArgType& src_type, const auto& src_inout = src_type == OpArgType::IN ? src_op->inputs_ : src_op->outputs_; auto& dst_inout = *vars; + const OpProto* proto = OpRegistry::op_info_map().at(src_op->type_).proto_; const auto& src_arg_list = - src_type == OpArgType::IN ? proto.inputs() : proto.outputs(); + src_type == OpArgType::IN ? proto->inputs() : proto->outputs(); for (const auto& arg : src_arg_list) { if (arg.no_gradient() && !is_grad) continue; const std::string src_name = arg.name(); @@ -43,6 +44,8 @@ OperatorBase* BuildGradOp(const OperatorBase* op) { auto it = OpRegistry::op_info_map().find(op->type_); PADDLE_ENFORCE(it != OpRegistry::op_info_map().end(), "'%s' has not been registered.", op->type_); + PADDLE_ENFORCE(it->second.proto_ != nullptr, "'%s' has no OpProto.", + op->type_); std::string grad_op_type = it->second.grad_op_type_; PADDLE_ENFORCE(!grad_op_type.empty(), "'%s' has no gradient operator.", op->type_); diff --git a/paddle/framework/op_registry.h b/paddle/framework/op_registry.h index 6dd5f4af22..120f4ede6b 100644 --- a/paddle/framework/op_registry.h +++ b/paddle/framework/op_registry.h @@ -126,13 +126,6 @@ class NOPMaker : public OpProtoAndCheckerMaker { : OpProtoAndCheckerMaker(proto, op_checker) {} }; -struct OpInfo { - std::function creator_; - std::string grad_op_type_; - OpProto* proto_; - OpAttrChecker* checker_; -}; - class OpRegistry { using VarNameMap = OperatorBase::VarNameMap; using OpCreator = std::function; public: + struct OpInfo { + OpCreator creator_; + std::string grad_op_type_; + OpProto* proto_; + OpAttrChecker* checker_; + }; + template static void RegisterOp(const std::string& op_type, const std::string& grad_op_type) { @@ -175,9 +175,9 @@ class OpRegistry { } static std::shared_ptr CreateOp(const std::string& type, - const VarNameList& inputs, - const VarNameList& outputs, - const AttributeMap& attrs) { + const VarNameMap& inputs, + const VarNameMap& outputs, + AttributeMap attrs) { auto it = op_info_map().find(type); PADDLE_ENFORCE(it != op_info_map().end(), "Operator '%s' has not been registered.", type); diff --git a/paddle/framework/operator.cc b/paddle/framework/operator.cc index c054804477..0daf12e7f5 100644 --- a/paddle/framework/operator.cc +++ b/paddle/framework/operator.cc @@ -152,7 +152,7 @@ std::vector OperatorBase::OutputVars(bool has_intermediate) const { type_); // get all OpProto::Var for outputs - for (auto& o : it->second.proto_.outputs()) { + for (auto& o : it->second.proto_->outputs()) { // ignore all intermediate output if (o.intermediate()) continue; auto out = outputs_.find(o.name()); From 39c986c73029b39576772afd4806f37fdfe9d57a Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Mon, 14 Aug 2017 13:47:15 -0700 Subject: [PATCH 11/12] restart CI --- paddle/framework/operator.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/framework/operator.h b/paddle/framework/operator.h index 606ab9e502..144db220a2 100644 --- a/paddle/framework/operator.h +++ b/paddle/framework/operator.h @@ -216,7 +216,7 @@ class InferShapeContext { [&](const std::string& sub_name) { auto var = scope_.FindVar(sub_name); PADDLE_ENFORCE_NOT_NULL( - var, "MultiOutput(%s:%s) should not be nullptr", name, + var, "MultiOutput(%s:%s) should not be nullptr.", name, sub_name); return var->GetMutable(); }); From 914a2f2b42c28b5a037b470c613b305303d99e16 Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Mon, 14 Aug 2017 21:30:31 +0000 Subject: [PATCH 12/12] Format code --- paddle/framework/grad_op_builder_test.cc | 14 ++++++-------- paddle/framework/pybind.cc | 4 ++-- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/paddle/framework/grad_op_builder_test.cc b/paddle/framework/grad_op_builder_test.cc index 043503ada6..d0d5d64fe6 100644 --- a/paddle/framework/grad_op_builder_test.cc +++ b/paddle/framework/grad_op_builder_test.cc @@ -59,10 +59,9 @@ REGISTER_OP(io_ignored, f::NOP, f::IOIgnoredOpMaker, io_ignored_grad, f::NOP); TEST(GradOpBuilder, MutiInOut) { std::shared_ptr test_op(f::OpRegistry::CreateOp( - "mult_io", - {{"In1", {"in1"}}, - {"In2_mult", {"in2_1", "in2_2", "in2_3"}}, - {"In3", {"in3"}}}, + "mult_io", {{"In1", {"in1"}}, + {"In2_mult", {"in2_1", "in2_2", "in2_3"}}, + {"In3", {"in3"}}}, {{"Out1", {"out1"}}, {"Out2_mult", {"out2_1", "out2_2"}}}, {})); std::shared_ptr grad_test_op = f::OpRegistry::CreateGradOp(*test_op); @@ -92,10 +91,9 @@ TEST(GradOpBuilder, MutiInOut) { TEST(GradOpBuilder, IOIgnoredInGradient) { std::shared_ptr test_op(f::OpRegistry::CreateOp( - "io_ignored", - {{"In1", {"in1"}}, - {"In2_mult", {"in2_1", "in2_2"}}, - {"In3_mult", {"in3_1", "in3_2"}}}, + "io_ignored", {{"In1", {"in1"}}, + {"In2_mult", {"in2_1", "in2_2"}}, + {"In3_mult", {"in3_1", "in3_2"}}}, {{"Out1_mult", {"out1_1", "out1_2"}}, {"Out2", {"out2"}}}, {})); std::shared_ptr grad_test_op = f::OpRegistry::CreateGradOp(*test_op); diff --git a/paddle/framework/pybind.cc b/paddle/framework/pybind.cc index 108ae79d2c..047e09642c 100644 --- a/paddle/framework/pybind.cc +++ b/paddle/framework/pybind.cc @@ -58,8 +58,8 @@ void ExposeOperator(ClassType &m) { .def("outputs", [](const typename ClassType::type &op) -> std::map> { - return op.outputs_; - }) + return op.outputs_; + }) .def("inputs", [](const typename ClassType::type &op) { return op.inputs_; }) .def("__str__", &ClassType::type::DebugString)