From 264b644718c14da348114bb9a44afddcd7166f11 Mon Sep 17 00:00:00 2001 From: dongzhihong Date: Wed, 2 Aug 2017 21:26:29 +0800 Subject: [PATCH 01/10] "add rowwise add backward op" --- paddle/operators/rowwise_add_op.cc | 15 +++++++++++++++ paddle/operators/rowwise_add_op.h | 19 +++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/paddle/operators/rowwise_add_op.cc b/paddle/operators/rowwise_add_op.cc index 2ad2b66c8f..cc763a8cf4 100644 --- a/paddle/operators/rowwise_add_op.cc +++ b/paddle/operators/rowwise_add_op.cc @@ -46,6 +46,17 @@ for i in xrange(X.shape[0]): )DOC"); } }; +class RowWiseAddGradOp : public OperatorWithKernel { +protected: + void InferShape(const InferShapeContext &ctx) const override { + PADDLE_ENFORCE(ctx.InputSize() == 4UL, + "RowWiseAddGrad inputs is I, O, OG, size must be 4"); + PADDLE_ENFORCE(ctx.OutputSize() == 2, + "RowWiseAddGrad output is IG, size must be 2"); + ctx.Output(0)->Resize(ctx.Input(0)->dims()); + ctx.Output(1)->Resize(ctx.Input(1)->dims()); + } +}; } // namespace operators } // namespace paddle @@ -53,3 +64,7 @@ for i in xrange(X.shape[0]): REGISTER_OP(rowwise_add, ops::RowWiseAddOp, ops::RowWiseAddOpMaker); REGISTER_OP_CPU_KERNEL(rowwise_add, ops::RowWiseAddKernel); + +REGISTER_GRADIENT_OP(rowwise_add, rowwise_add_grad, ops::RowWiseAddGradOp); +REGISTER_OP_CPU_KERNEL(rowwise_add_grad, + ops::RowWiseAddGradKernel); diff --git a/paddle/operators/rowwise_add_op.h b/paddle/operators/rowwise_add_op.h index b86dd54634..940459e0f1 100644 --- a/paddle/operators/rowwise_add_op.h +++ b/paddle/operators/rowwise_add_op.h @@ -38,5 +38,24 @@ public: } }; +template +class RowWiseAddGradKernel : public OpKernel { +public: + void Compute(const ExecutionContext& context) const override { + auto XGrad = context.Output(0); + auto bGrad = context.Output(1); + XGrad->mutable_data(context.GetPlace()); + bGrad->mutable_data(context.GetPlace()); + + // I, O, OG => [X, b], [Out], [OutGrad] + auto OutGrad = EigenMatrix::From(*context.Input(3)); + EigenMatrix::From(*XGrad).device(*(context.GetEigenDevice())) = + OutGrad; + // const int dimension = bGrad.dimension(0); + // https://eigen.tuxfamily.org/dox/unsupported/TensorBase_8h_source.html + EigenVector::Flatten(*bGrad).device(*(context.GetEigenDevice())) = + OutGrad.cumsum(1); // colwise add + } +}; } // namespace operators } // namespace paddle From 8ff3590eda2a6488f4b06f5ce6ffe553ae42d0a6 Mon Sep 17 00:00:00 2001 From: dongzhihong Date: Fri, 4 Aug 2017 01:15:56 +0800 Subject: [PATCH 02/10] fix op name --- paddle/operators/rowwise_add_op.cc | 20 ++++++++++---------- paddle/operators/rowwise_add_op.cu | 2 +- paddle/operators/rowwise_add_op.h | 6 +++--- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/paddle/operators/rowwise_add_op.cc b/paddle/operators/rowwise_add_op.cc index cc763a8cf4..178ea3c614 100644 --- a/paddle/operators/rowwise_add_op.cc +++ b/paddle/operators/rowwise_add_op.cc @@ -16,7 +16,7 @@ namespace paddle { namespace operators { -class RowWiseAddOp : public OperatorWithKernel { +class RowwiseAddOp : public OperatorWithKernel { protected: void InferShape(const InferShapeContext &ctx) const override { PADDLE_ENFORCE(ctx.InputSize() == 2UL, @@ -32,9 +32,9 @@ protected: } }; -class RowWiseAddOpMaker : public OpProtoAndCheckerMaker { +class RowwiseAddOpMaker : public OpProtoAndCheckerMaker { public: - RowWiseAddOpMaker(OpProto *proto, OpAttrChecker *op_checker) + RowwiseAddOpMaker(OpProto *proto, OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "The left input of row-wise add op, must be matrix"); AddInput("b", "The right input of row-wise add op, must be vector"); @@ -46,13 +46,13 @@ for i in xrange(X.shape[0]): )DOC"); } }; -class RowWiseAddGradOp : public OperatorWithKernel { +class RowwiseAddGradOp : public OperatorWithKernel { protected: void InferShape(const InferShapeContext &ctx) const override { PADDLE_ENFORCE(ctx.InputSize() == 4UL, - "RowWiseAddGrad inputs is I, O, OG, size must be 4"); + "RowwiseAddGrad inputs is I, O, OG, size must be 4"); PADDLE_ENFORCE(ctx.OutputSize() == 2, - "RowWiseAddGrad output is IG, size must be 2"); + "RowwiseAddGrad output is IG, size must be 2"); ctx.Output(0)->Resize(ctx.Input(0)->dims()); ctx.Output(1)->Resize(ctx.Input(1)->dims()); } @@ -61,10 +61,10 @@ protected: } // namespace operators } // namespace paddle -REGISTER_OP(rowwise_add, ops::RowWiseAddOp, ops::RowWiseAddOpMaker); +REGISTER_OP(rowwise_add, ops::RowwiseAddOp, ops::RowwiseAddOpMaker); REGISTER_OP_CPU_KERNEL(rowwise_add, - ops::RowWiseAddKernel); + ops::RowwiseAddKernel); -REGISTER_GRADIENT_OP(rowwise_add, rowwise_add_grad, ops::RowWiseAddGradOp); +REGISTER_GRADIENT_OP(rowwise_add, rowwise_add_grad, ops::RowwiseAddGradOp); REGISTER_OP_CPU_KERNEL(rowwise_add_grad, - ops::RowWiseAddGradKernel); + ops::RowwiseAddGradKernel); diff --git a/paddle/operators/rowwise_add_op.cu b/paddle/operators/rowwise_add_op.cu index 4b33e38eba..f48dfeb6f2 100644 --- a/paddle/operators/rowwise_add_op.cu +++ b/paddle/operators/rowwise_add_op.cu @@ -1,4 +1,4 @@ #include "paddle/operators/rowwise_add_op.h" REGISTER_OP_GPU_KERNEL(rowwise_add, - ops::RowWiseAddKernel); + ops::RowwiseAddKernel); diff --git a/paddle/operators/rowwise_add_op.h b/paddle/operators/rowwise_add_op.h index 940459e0f1..321f51e61d 100644 --- a/paddle/operators/rowwise_add_op.h +++ b/paddle/operators/rowwise_add_op.h @@ -19,7 +19,7 @@ namespace paddle { namespace operators { template -class RowWiseAddKernel : public OpKernel { +class RowwiseAddKernel : public OpKernel { public: void Compute(const ExecutionContext& context) const override { auto out = context.Output(0); @@ -39,7 +39,7 @@ public: }; template -class RowWiseAddGradKernel : public OpKernel { +class RowwiseAddGradKernel : public OpKernel { public: void Compute(const ExecutionContext& context) const override { auto XGrad = context.Output(0); @@ -51,7 +51,7 @@ public: auto OutGrad = EigenMatrix::From(*context.Input(3)); EigenMatrix::From(*XGrad).device(*(context.GetEigenDevice())) = OutGrad; - // const int dimension = bGrad.dimension(0); + // https://eigen.tuxfamily.org/dox/unsupported/TensorBase_8h_source.html EigenVector::Flatten(*bGrad).device(*(context.GetEigenDevice())) = OutGrad.cumsum(1); // colwise add From b7ee1e7d9c7f01844b23c54a3c5a2584e0a6a410 Mon Sep 17 00:00:00 2001 From: dongzhihong Date: Wed, 9 Aug 2017 00:12:09 +0800 Subject: [PATCH 03/10] "backward check todo" --- paddle/operators/rowwise_add_op.h | 8 ++++---- python/paddle/v2/framework/tests/test_rowwise_add_op.py | 2 ++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/paddle/operators/rowwise_add_op.h b/paddle/operators/rowwise_add_op.h index 06af88a993..965c0df532 100644 --- a/paddle/operators/rowwise_add_op.h +++ b/paddle/operators/rowwise_add_op.h @@ -42,18 +42,18 @@ template class RowwiseAddGradKernel : public OpKernel { public: void Compute(const ExecutionContext& context) const override { - auto XGrad = context.Output(0); - auto bGrad = context.Output(1); + auto* XGrad = context.Output(0); + auto* bGrad = context.Output(1); XGrad->mutable_data(context.GetPlace()); bGrad->mutable_data(context.GetPlace()); // I, O, OG => [X, b], [Out], [OutGrad] auto OutGrad = EigenMatrix::From(*context.Input(3)); - EigenMatrix::From(*XGrad).device(*(context.GetEigenDevice())) = + EigenMatrix::From(*XGrad).device(context.GetEigenDevice()) = OutGrad; // https://eigen.tuxfamily.org/dox/unsupported/TensorBase_8h_source.html - EigenVector::Flatten(*bGrad).device(*(context.GetEigenDevice())) = + EigenVector::Flatten(*bGrad).device(context.GetEigenDevice()) = OutGrad.cumsum(1); // colwise add } }; diff --git a/python/paddle/v2/framework/tests/test_rowwise_add_op.py b/python/paddle/v2/framework/tests/test_rowwise_add_op.py index f8521eb517..e957dd6b3f 100644 --- a/python/paddle/v2/framework/tests/test_rowwise_add_op.py +++ b/python/paddle/v2/framework/tests/test_rowwise_add_op.py @@ -15,5 +15,7 @@ class TestRowwiseAddOp(unittest.TestCase): self.outputs = {'Out': np.add(self.inputs['X'], self.inputs['b'])} +#TODO(dzh): rowwise_grad check + if __name__ == '__main__': unittest.main() From 200e3e2c6b7b3c1be47204f0e76ab79696f46efb Mon Sep 17 00:00:00 2001 From: dongzhihong Date: Wed, 9 Aug 2017 10:14:20 +0800 Subject: [PATCH 04/10] "change namespace prefix" --- paddle/operators/rowwise_add_op.cc | 6 +++--- paddle/operators/rowwise_add_op.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/paddle/operators/rowwise_add_op.cc b/paddle/operators/rowwise_add_op.cc index c192da04da..a012ab0be0 100644 --- a/paddle/operators/rowwise_add_op.cc +++ b/paddle/operators/rowwise_add_op.cc @@ -35,7 +35,7 @@ class RowwiseAddOp : public framework::OperatorWithKernel { class RowwiseAddOpMaker : public framework::OpProtoAndCheckerMaker { public: - RowWiseAddOpMaker(framework::OpProto *proto, + RowwiseAddOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "The left input of row-wise add op, must be matrix"); @@ -48,9 +48,9 @@ for i in xrange(X.shape[0]): )DOC"); } }; -class RowwiseAddGradOp : public OperatorWithKernel { +class RowwiseAddGradOp : public framework::OperatorWithKernel { protected: - void InferShape(const InferShapeContext &ctx) const override { + void InferShape(const framework::InferShapeContext &ctx) const override { PADDLE_ENFORCE(ctx.InputSize() == 4UL, "RowwiseAddGrad inputs is I, O, OG, size must be 4"); PADDLE_ENFORCE(ctx.OutputSize() == 2, diff --git a/paddle/operators/rowwise_add_op.h b/paddle/operators/rowwise_add_op.h index ad43e753e4..27d7a33e8a 100644 --- a/paddle/operators/rowwise_add_op.h +++ b/paddle/operators/rowwise_add_op.h @@ -28,7 +28,7 @@ template ; template -class RowwiseAddKernel : public OpKernel { +class RowwiseAddKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { auto out = context.Output(0); From 68bfc3ff963474e12c8af1c3575128b0acac90ed Mon Sep 17 00:00:00 2001 From: dongzhihong Date: Wed, 9 Aug 2017 21:02:51 +0800 Subject: [PATCH 05/10] "add python test" --- .../v2/framework/tests/test_rowwise_add_op.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/python/paddle/v2/framework/tests/test_rowwise_add_op.py b/python/paddle/v2/framework/tests/test_rowwise_add_op.py index e957dd6b3f..1b27f54f15 100644 --- a/python/paddle/v2/framework/tests/test_rowwise_add_op.py +++ b/python/paddle/v2/framework/tests/test_rowwise_add_op.py @@ -1,6 +1,7 @@ import unittest -from op_test_util import OpTestMeta import numpy as np +from op_test_util import OpTestMeta +from gradient_checker import GradientChecker, create_op class TestRowwiseAddOp(unittest.TestCase): @@ -15,6 +16,16 @@ class TestRowwiseAddOp(unittest.TestCase): self.outputs = {'Out': np.add(self.inputs['X'], self.inputs['b'])} +class RowwiseAddGradOpTest(GradientChecker): + def test_rowwise_add(self): + op = create_op("rowwise_add") + inputs = { + "X": np.random.uniform(0.1, 1, [10, 10]).astype("float32"), + "b": np.random.uniform(0.1, 1, [10, 1]).astype("float32") + } + self.check_grad(op, inputs, set("X", "b"), "Out") + + #TODO(dzh): rowwise_grad check if __name__ == '__main__': From 7c0cb0c7901093e7b2aa57100f086f737ab39739 Mon Sep 17 00:00:00 2001 From: dongzhihong Date: Wed, 9 Aug 2017 23:51:46 +0800 Subject: [PATCH 06/10] "fix ci launch" --- python/paddle/v2/framework/tests/test_rowwise_add_op.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/v2/framework/tests/test_rowwise_add_op.py b/python/paddle/v2/framework/tests/test_rowwise_add_op.py index 1b27f54f15..8118d2d741 100644 --- a/python/paddle/v2/framework/tests/test_rowwise_add_op.py +++ b/python/paddle/v2/framework/tests/test_rowwise_add_op.py @@ -23,7 +23,7 @@ class RowwiseAddGradOpTest(GradientChecker): "X": np.random.uniform(0.1, 1, [10, 10]).astype("float32"), "b": np.random.uniform(0.1, 1, [10, 1]).astype("float32") } - self.check_grad(op, inputs, set("X", "b"), "Out") + self.check_grad(op, inputs, set(["X", "b"]), "Out") #TODO(dzh): rowwise_grad check From 12ee5014857e751fb429e0d3ebcfd41dcd5da29d Mon Sep 17 00:00:00 2001 From: dongzhihong Date: Mon, 14 Aug 2017 20:57:46 +0800 Subject: [PATCH 07/10] "fix operator grad config" --- paddle/operators/rowwise_add_op.cc | 23 +++++++++++++++++------ paddle/operators/rowwise_add_op.h | 21 +++++++++++---------- 2 files changed, 28 insertions(+), 16 deletions(-) diff --git a/paddle/operators/rowwise_add_op.cc b/paddle/operators/rowwise_add_op.cc index 0c6ae64d0c..60e5d7749c 100644 --- a/paddle/operators/rowwise_add_op.cc +++ b/paddle/operators/rowwise_add_op.cc @@ -17,6 +17,8 @@ namespace paddle { namespace operators { +using framework::Tensor; + class RowwiseAddOp : public framework::OperatorWithKernel { public: using framework::OperatorWithKernel::OperatorWithKernel; @@ -50,14 +52,23 @@ for i in xrange(X.shape[0]): } }; class RowwiseAddGradOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + protected: void InferShape(const framework::InferShapeContext &ctx) const override { - PADDLE_ENFORCE(ctx.InputSize() == 4UL, - "RowwiseAddGrad inputs is I, O, OG, size must be 4"); - PADDLE_ENFORCE(ctx.OutputSize() == 2, - "RowwiseAddGrad output is IG, size must be 2"); - ctx.Output(0)->Resize(ctx.Input(0)->dims()); - ctx.Output(1)->Resize(ctx.Input(1)->dims()); + // PADDLE_ENFORCE(ctx.InputSize() == 4UL, + // "RowwiseAddGrad inputs is I, O, OG, size must be 4"); + // PADDLE_ENFORCE(ctx.OutputSize() == 2, + // "RowwiseAddGrad output is IG, size must be 2"); + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), "X should not be null"); + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("b"), "b should not be null"); + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar(framework::GradVarName("Out")), + "Input(Out@GRAD) should not be null"); + auto dims0 = ctx.Input("X")->dims(); + auto dims1 = ctx.Input("b")->dims(); + ctx.Output(framework::GradVarName("X"))->Resize(dims0); + ctx.Output(framework::GradVarName("b"))->Resize(dims1); } }; diff --git a/paddle/operators/rowwise_add_op.h b/paddle/operators/rowwise_add_op.h index 3ad60172c1..6593d811e4 100644 --- a/paddle/operators/rowwise_add_op.h +++ b/paddle/operators/rowwise_add_op.h @@ -51,19 +51,20 @@ template class RowwiseAddGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { - auto* XGrad = context.Output(0); - auto* bGrad = context.Output(1); - XGrad->mutable_data(context.GetPlace()); - bGrad->mutable_data(context.GetPlace()); + auto* dX = context.Output(framework::GradVarName("X")); + auto* db = context.Output(framework::GradVarName("b")); + auto* dOut = context.Output(framework::GradVarName("Out")); + dX->mutable_data(context.GetPlace()); + db->mutable_data(context.GetPlace()); - // I, O, OG => [X, b], [Out], [OutGrad] - auto OutGrad = EigenMatrix::From(*context.Input(3)); - EigenMatrix::From(*XGrad).device(context.GetEigenDevice()) = - OutGrad; + auto OutGrad = EigenMatrix::From(*dOut); + auto place = context.GetEigenDevice(); + EigenMatrix::From(*dX).device(place) = OutGrad; // https://eigen.tuxfamily.org/dox/unsupported/TensorBase_8h_source.html - EigenVector::Flatten(*bGrad).device(context.GetEigenDevice()) = - OutGrad.cumsum(1); // colwise add + // colwise add + Eigen::array dims{{1}}; /* dimension to reduce */ + EigenVector::Flatten(*db).device(place) = OutGrad.sum(dims); } }; } // namespace operators From c332e4ee25ca28f307c1d3ccbcec9458fd25f5b3 Mon Sep 17 00:00:00 2001 From: dongzhihong Date: Thu, 17 Aug 2017 16:12:27 -0700 Subject: [PATCH 08/10] "relauch the ci" --- paddle/operators/rowwise_add_op.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/operators/rowwise_add_op.cc b/paddle/operators/rowwise_add_op.cc index 15192d90be..82e5df591d 100644 --- a/paddle/operators/rowwise_add_op.cc +++ b/paddle/operators/rowwise_add_op.cc @@ -73,7 +73,7 @@ class RowwiseAddGradOp : public framework::OperatorWithKernel { namespace ops = paddle::operators; REGISTER_OP(rowwise_add, ops::RowwiseAddOp, ops::RowwiseAddOpMaker, - rowwise_add_grad); + rowwise_add_grad, ops::RowwiseAddGradOp); REGISTER_OP_CPU_KERNEL( rowwise_add, ops::RowwiseAddKernel); REGISTER_OP_CPU_KERNEL( From cef27dab47b430ce4034cfcfedf0c6bc95266f51 Mon Sep 17 00:00:00 2001 From: dongzhihong Date: Thu, 17 Aug 2017 19:14:27 -0700 Subject: [PATCH 09/10] "add fixl" --- paddle/operators/rowwise_add_op.cc | 1 + python/paddle/v2/framework/tests/test_rowwise_add_op.py | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/paddle/operators/rowwise_add_op.cc b/paddle/operators/rowwise_add_op.cc index 82e5df591d..f07dd8f602 100644 --- a/paddle/operators/rowwise_add_op.cc +++ b/paddle/operators/rowwise_add_op.cc @@ -63,6 +63,7 @@ class RowwiseAddGradOp : public framework::OperatorWithKernel { "Input(Out@GRAD) should not be null"); auto dims0 = ctx.Input("X")->dims(); auto dims1 = ctx.Input("b")->dims(); + PADDLE_ENFORCE_EQ(1, framework::product(dims1), "b dims should be 1") ctx.Output(framework::GradVarName("X"))->Resize(dims0); ctx.Output(framework::GradVarName("b"))->Resize(dims1); } diff --git a/python/paddle/v2/framework/tests/test_rowwise_add_op.py b/python/paddle/v2/framework/tests/test_rowwise_add_op.py index 8118d2d741..29d72e8500 100644 --- a/python/paddle/v2/framework/tests/test_rowwise_add_op.py +++ b/python/paddle/v2/framework/tests/test_rowwise_add_op.py @@ -21,12 +21,10 @@ class RowwiseAddGradOpTest(GradientChecker): op = create_op("rowwise_add") inputs = { "X": np.random.uniform(0.1, 1, [10, 10]).astype("float32"), - "b": np.random.uniform(0.1, 1, [10, 1]).astype("float32") + "b": np.random.uniform(0.1, 1, [10]).astype("float32") } self.check_grad(op, inputs, set(["X", "b"]), "Out") -#TODO(dzh): rowwise_grad check - if __name__ == '__main__': unittest.main() From 82b820e97b90f21d7b46629bba72436a69e888e1 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Fri, 18 Aug 2017 08:21:56 -0700 Subject: [PATCH 10/10] fix rowwise_add_grad_op --- paddle/operators/rowwise_add_op.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/operators/rowwise_add_op.cc b/paddle/operators/rowwise_add_op.cc index f07dd8f602..6825dce332 100644 --- a/paddle/operators/rowwise_add_op.cc +++ b/paddle/operators/rowwise_add_op.cc @@ -63,7 +63,7 @@ class RowwiseAddGradOp : public framework::OperatorWithKernel { "Input(Out@GRAD) should not be null"); auto dims0 = ctx.Input("X")->dims(); auto dims1 = ctx.Input("b")->dims(); - PADDLE_ENFORCE_EQ(1, framework::product(dims1), "b dims should be 1") + PADDLE_ENFORCE_EQ(1, dims1.size(), "b dims should be 1") ctx.Output(framework::GradVarName("X"))->Resize(dims0); ctx.Output(framework::GradVarName("b"))->Resize(dims1); }