From 2763f3e32f7f37d52cbd6379b036958ad3d34ad1 Mon Sep 17 00:00:00 2001 From: yangyaming Date: Wed, 6 Sep 2017 15:32:54 +0800 Subject: [PATCH 01/35] Complete smooth_l1_loss_op. --- paddle/operators/smooth_l1_loss_op.cc | 119 +++++++++++ paddle/operators/smooth_l1_loss_op.cu | 24 +++ paddle/operators/smooth_l1_loss_op.h | 184 ++++++++++++++++++ paddle/pybind/pybind.cc | 1 + .../paddle/v2/framework/tests/CMakeLists.txt | 1 + .../framework/tests/test_smooth_l1_loss_op.py | 106 ++++++++++ 6 files changed, 435 insertions(+) create mode 100644 paddle/operators/smooth_l1_loss_op.cc create mode 100644 paddle/operators/smooth_l1_loss_op.cu create mode 100644 paddle/operators/smooth_l1_loss_op.h create mode 100644 python/paddle/v2/framework/tests/test_smooth_l1_loss_op.py diff --git a/paddle/operators/smooth_l1_loss_op.cc b/paddle/operators/smooth_l1_loss_op.cc new file mode 100644 index 0000000000..e9a3847417 --- /dev/null +++ b/paddle/operators/smooth_l1_loss_op.cc @@ -0,0 +1,119 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/smooth_l1_loss_op.h" + +namespace paddle { +namespace operators { + +class SmoothL1LossOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(const framework::InferShapeContext& ctx) const override { + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), + "Input of SmoothL1LossOp must be initialized."); + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Y"), + "Target of SmoothL1LossOp must be initialized."); + + auto* x = ctx.Input("X"); + auto* y = ctx.Input("Y"); + PADDLE_ENFORCE_EQ(x->dims(), y->dims(), + "Dimensions of SmoothL1LossOp's input and target " + "must be same."); + PADDLE_ENFORCE_GE(framework::arity(x->dims()), 2, + "Tensor rank of SmoothL1LossOp's input must be " + "at least 2."); + auto* inside_weight = ctx.Input("InsideWeight"); + if (inside_weight) { + auto* outside_weight = ctx.Input("OutsideWeight"); + PADDLE_ENFORCE_NOT_NULL(outside_weight, + "If weights are provided, must specify both " + "inside and outside weights."); + PADDLE_ENFORCE_EQ(inside_weight->dims(), x->dims(), + "Dimensions of inside weight must be same with input."); + PADDLE_ENFORCE_EQ( + outside_weight->dims(), x->dims(), + "Dimensions of outside weight must be same with input."); + } + + auto* diff = ctx.Output("diff"); + auto* out = ctx.Output("Out"); + diff->Resize(x->dims()); + // loss is a two-rank tensor + out->Resize({x->dims()[0], 1}); + } +}; + +template +class SmoothL1LossOpMaker : public framework::OpProtoAndCheckerMaker { + public: + SmoothL1LossOpMaker(framework::OpProto* proto, + framework::OpAttrChecker* op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "Input of SmoothL1LossOp."); + AddInput("Y", "Target of SmoothL1LossOp."); + AddInput("InsideWeight", "Optional input to scale (X-Y)."); + AddInput("OutsideWeight", "Optinal input to scale smooth l1 loss."); + AddOutput("diff", "Intermediate variable to cache Win*(X-Y).") + .AsIntermediate(); + AddOutput("Out", "Final smooth l1 loss of inputs."); + AddComment(R"DOC( +Compute SmoothL1Loss for input and target. + +The equation is: Out = 0.5 * (sigma * (X - Y)) ^ 2 if abs(X - Y) < 1 / sigma^2 + abs(X - Y) - 0.5 / sigma^2 otherwise +)DOC"); + AddAttr("sigma", "Hyper parameter, default value is 3.0 .") + .SetDefault(3.0); + } +}; + +class SmoothL1LossGradOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(const framework::InferShapeContext& ctx) const override { + auto in_dims = ctx.Input("X")->dims(); + auto out_dims = + ctx.Input(framework::GradVarName("Out"))->dims(); + auto* x_grad = ctx.Output(framework::GradVarName("X")); + auto* y_grad = ctx.Output(framework::GradVarName("Y")); + + PADDLE_ENFORCE_GE(framework::arity(out_dims), 2, + "Tensor rank of output gradient should be 2."); + PADDLE_ENFORCE_EQ(out_dims[0], in_dims[0], + "First dimension of ouptut gradient must be " + "same with input."); + PADDLE_ENFORCE_EQ(out_dims[1], 1, + "Second dimension of output gradient must be 1."); + + if (x_grad) x_grad->Resize(in_dims); + if (y_grad) y_grad->Resize(in_dims); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP(smooth_l1_loss, ops::SmoothL1LossOp, + ops::SmoothL1LossOpMaker, ops::SmoothL1LossGradOp); +REGISTER_OP_CPU_KERNEL( + smooth_l1_loss, ops::SmoothL1LossKernel); +REGISTER_OP_CPU_KERNEL( + smooth_l1_loss_grad, + ops::SmoothL1LossGradKernel); diff --git a/paddle/operators/smooth_l1_loss_op.cu b/paddle/operators/smooth_l1_loss_op.cu new file mode 100644 index 0000000000..1c3172f438 --- /dev/null +++ b/paddle/operators/smooth_l1_loss_op.cu @@ -0,0 +1,24 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#define EIGEN_USE_GPU + +#include "paddle/operators/smooth_l1_loss_op.h" + +namespace ops = paddle::operators; +REGISTER_OP_GPU_KERNEL( + smooth_l1_loss, ops::SmoothL1LossKernel); +REGISTER_OP_GPU_KERNEL( + smooth_l1_loss_grad, + ops::SmoothL1LossGradKernel); diff --git a/paddle/operators/smooth_l1_loss_op.h b/paddle/operators/smooth_l1_loss_op.h new file mode 100644 index 0000000000..ae91b9c893 --- /dev/null +++ b/paddle/operators/smooth_l1_loss_op.h @@ -0,0 +1,184 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once +#include "paddle/framework/eigen.h" +#include "paddle/framework/op_registry.h" + +namespace paddle { +namespace operators { + +using Tensor = framework::Tensor; +template +using EigenVector = framework::EigenVector; +template +using EigenMatrix = framework::EigenMatrix; + +template +struct SmoothL1LossFoward { + __host__ __device__ SmoothL1LossFoward(const T& sigma2) : sigma2(sigma2) {} + + __host__ __device__ T operator()(const T& val) const { + T abs_val = std::abs(val); + if (abs_val < 1.0 / sigma2) { + return 0.5 * val * val * sigma2; + } else { + return abs_val - 0.5 / sigma2; + } + } + + T sigma2; +}; + +template +class SmoothL1LossKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* in0 = context.Input("X"); + auto* in1 = context.Input("Y"); + auto* in2 = context.Input("InsideWeight"); + auto* in3 = context.Input("OutsideWeight"); + auto* out0 = context.Output("diff"); + auto* out1 = context.Output("Out"); + + out0->mutable_data(context.GetPlace()); + out1->mutable_data(context.GetPlace()); + auto place = context.GetEigenDevice(); + + auto sigma = static_cast(context.op_.GetAttr("sigma")); + T sigma2 = sigma * sigma; + bool has_weight = (in2 != nullptr) && (in3 != nullptr); + + auto x = EigenVector::Flatten(*in0); + auto y = EigenVector::Flatten(*in1); + auto diff = EigenVector::Flatten(*out0); + + diff.device(place) = x - y; + // multiply inside weight + if (has_weight) { + auto inside_weight = EigenVector::Flatten(*in2); + // cache diff, reused in bp + diff.device(place) = diff * inside_weight; + } + + auto in_counts = framework::product(in0->dims()); + Tensor paddle_errors; + paddle_errors.mutable_data({static_cast(in_counts)}, + context.GetPlace()); + auto errors = EigenVector::Flatten(paddle_errors); + // apply smooth l1 forward + errors.device(place) = diff.unaryExpr(SmoothL1LossFoward(sigma2)); + + // multiply outside weight + if (has_weight) { + auto outside_weight = EigenVector::Flatten(*in3); + errors.device(place) = errors * outside_weight; + } + auto loss = EigenMatrix::From(*out1, {in0->dims()[0], 1}); + // first dimension of 'X' is the number of samples + auto errors_mat_view = EigenMatrix::From(paddle_errors, in0->dims()); + loss.device(place) = errors_mat_view.sum(Eigen::array({1})); + } +}; + +template +struct SmoothL1LossBackward { + __host__ __device__ SmoothL1LossBackward(const T& sigma2) : sigma2(sigma2) {} + + __host__ __device__ T operator()(const T& val) const { + T abs_val = std::abs(val); + if (abs_val < 1.0 / sigma2) { + return sigma2 * val; + } else { + return (0 < val) - (val < 0); + } + } + + T sigma2; +}; + +template +class SmoothL1LossGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* in0 = context.Input("InsideWeight"); + auto* in1 = context.Input("OutsideWeight"); + auto* in2 = context.Input("diff"); + auto* og = context.Input(framework::GradVarName("Out")); + auto sigma = static_cast(context.op_.GetAttr("sigma")); + T sigma2 = sigma * sigma; + bool has_weight = (in0 != nullptr) && (in1 != nullptr); + + auto place = context.GetEigenDevice(); + + auto in_dims = in2->dims(); + auto counts = framework::product(in_dims); + auto cols = counts / in_dims[0]; + auto mat_dims = framework::make_ddim( + {static_cast(in_dims[0]), static_cast(cols)}); + + Tensor paddle_diff; + paddle_diff.mutable_data({static_cast(counts)}, context.GetPlace()); + auto diff = EigenVector::Flatten(paddle_diff); + // apply smooth l1 backwoard + diff.device(place) = EigenVector::Flatten(*in2).unaryExpr( + SmoothL1LossBackward(sigma2)); + + auto* out0 = context.Output(framework::GradVarName("X")); + auto* out1 = context.Output(framework::GradVarName("Y")); + + // compute weights + Tensor paddle_weights; + paddle_weights.mutable_data(mat_dims, context.GetPlace()); + auto weights = EigenMatrix::From(paddle_weights); + // initialize to 1.0 + if (platform::is_cpu_place(context.GetPlace())) { + weights.setConstant(static_cast(1.0)); + } else { + Tensor paddle_cpu_weights; + paddle_cpu_weights.mutable_data(mat_dims, platform::CPUPlace()); + EigenMatrix::From(paddle_cpu_weights).setConstant(static_cast(1.0)); + paddle_weights.CopyFrom(paddle_cpu_weights, context.GetPlace()); + } + if (has_weight) { + auto inside_weight = EigenMatrix::From(*in0, mat_dims); + auto outside_weight = EigenMatrix::From(*in1, mat_dims); + weights.device(place) = inside_weight * outside_weight; + } + + // compute gradients + auto out_grad = EigenMatrix::From(*og); + auto diff_mat_view = EigenMatrix::From(paddle_diff, mat_dims); + auto gradients = + out_grad.broadcast(Eigen::array({1, static_cast(cols)})) * + weights * diff_mat_view; + + if (out0) { + out0->mutable_data(context.GetPlace()); + auto x_grad = EigenMatrix::From(*out0, mat_dims); + x_grad.device(place) = gradients; + } + + if (out1) { + out1->mutable_data(context.GetPlace()); + auto y_grad = EigenMatrix::From(*out1, mat_dims); + y_grad.device(place) = -1 * gradients; + } + } +}; + +} // namespace operators +} // namespace paddle diff --git a/paddle/pybind/pybind.cc b/paddle/pybind/pybind.cc index 3bc150ccb7..5aaa372664 100644 --- a/paddle/pybind/pybind.cc +++ b/paddle/pybind/pybind.cc @@ -48,6 +48,7 @@ USE_OP_ITSELF(identity); USE_OP(minus); USE_CPU_ONLY_OP(gather); USE_CPU_ONLY_OP(scatter); +USE_OP(smooth_l1_loss); namespace paddle { namespace framework { diff --git a/python/paddle/v2/framework/tests/CMakeLists.txt b/python/paddle/v2/framework/tests/CMakeLists.txt index 661ebd8964..763f3a9f95 100644 --- a/python/paddle/v2/framework/tests/CMakeLists.txt +++ b/python/paddle/v2/framework/tests/CMakeLists.txt @@ -32,3 +32,4 @@ py_test(test_gradient_checker SRCS test_gradient_checker.py) py_test(test_lookup_table SRCS test_lookup_table.py) py_test(test_scale_and_identity_op SRCS test_scale_and_identity_op.py) py_test(mnist SRCS mnist.py) +py_test(test_smooth_l1_loss_op SRCS test_smooth_l1_loss_op.py) diff --git a/python/paddle/v2/framework/tests/test_smooth_l1_loss_op.py b/python/paddle/v2/framework/tests/test_smooth_l1_loss_op.py new file mode 100644 index 0000000000..b3432e703e --- /dev/null +++ b/python/paddle/v2/framework/tests/test_smooth_l1_loss_op.py @@ -0,0 +1,106 @@ +import unittest +from op_test_util import OpTestMeta +from gradient_checker import GradientChecker, create_op +import functools +import numpy as np +from paddle.v2.framework.op import Operator + + +def smooth_l1_loss_forward(val, sigma2): + abs_val = abs(val) + if abs_val < 1.0 / sigma2: + return 0.5 * val * val * sigma2 + else: + return abs_val - 0.5 / sigma2 + + +class TestSmoothL1LossOp_f0(unittest.TestCase): + __metaclass__ = OpTestMeta + + def setUp(self): + self.type = "smooth_l1_loss" + dims = (32, 64) + self.inputs = { + 'X': np.random.random(dims).astype("float32"), + 'Y': np.random.random(dims).astype("float32") + } + sigma = 3.0 + self.attrs = {'sigma': sigma} + sigma2 = sigma * sigma + diff = self.inputs['X'] - self.inputs['Y'] + loss = np.vectorize(smooth_l1_loss_forward)(diff, sigma2).sum(1) + loss = loss.reshape((dims[0], 1)) + self.outputs = {'diff': diff, 'Out': loss} + + +class TestSmoothL1LossOp_f1(unittest.TestCase): + __metaclass__ = OpTestMeta + + def setUp(self): + self.type = "smooth_l1_loss" + dims = (32, 64) + self.inputs = { + 'X': np.random.random(dims).astype("float32"), + 'Y': np.random.random(dims).astype("float32"), + 'InsideWeight': np.random.random(dims).astype("float32"), + 'OutsideWeight': np.random.random(dims).astype("float32") + } + sigma = 3.0 + self.attrs = {'sigma': sigma} + sigma2 = sigma * sigma + diff = self.inputs['X'] - self.inputs['Y'] + diff = diff * self.inputs['InsideWeight'] + loss = np.vectorize(smooth_l1_loss_forward)(diff, sigma2) + loss = loss * self.inputs['OutsideWeight'] + loss = loss.sum(1).reshape((dims[0], 1)) + self.outputs = {'diff': diff, 'Out': loss} + + +class SmoothL1LossGradOpTest(GradientChecker): + def test_smooth_l1_loss_b0(self): + dims = (5, 7) + X = np.random.random(dims).astype("float32") + Y = np.random.random(dims).astype("float32") + InsideWeight = np.random.random(dims).astype("float32") + OutsideWeight = np.random.random(dims).astype("float32") + inputs = { + 'X': X, + 'Y': Y, + 'InsideWeight': InsideWeight, + 'OutsideWeight': OutsideWeight + } + op = Operator( + "smooth_l1_loss", + X='X', + Y='Y', + InsideWeight='InsideWeight', + OutsideWeight='OutsideWeight', + diff="diff", + Out="Out", + sigma=3.0) + self.compare_grad( + op, inputs, no_grad_set=set(['InsideWeight', 'OutsideWeight'])) + self.check_grad( + op, inputs, set(["X", "Y"]), "Out", max_relative_error=0.08) + + def test_smooth_l1_loss_b1(self): + dims = (5, 7) + X = np.random.random(dims).astype("float32") + Y = np.random.random(dims).astype("float32") + inputs = {'X': X, 'Y': Y} + op = Operator( + "smooth_l1_loss", + X='X', + Y='Y', + InsideWeight='InsideWeight', + OutsideWeight='OutsideWeight', + diff="diff", + Out="Out", + sigma=3.0) + self.compare_grad( + op, inputs, no_grad_set=set(['InsideWeight', 'OutsideWeight'])) + self.check_grad(op, inputs, set(["X", "Y"]), "Out") + + +if __name__ == '__main__': + unittest.main() From b7776e66d609bd26d40a6338b9534005621e876c Mon Sep 17 00:00:00 2001 From: yangyaming Date: Wed, 6 Sep 2017 18:30:59 +0800 Subject: [PATCH 02/35] Fix dimension bugs. --- paddle/operators/smooth_l1_loss_op.h | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/paddle/operators/smooth_l1_loss_op.h b/paddle/operators/smooth_l1_loss_op.h index ae91b9c893..3e47403858 100644 --- a/paddle/operators/smooth_l1_loss_op.h +++ b/paddle/operators/smooth_l1_loss_op.h @@ -87,10 +87,13 @@ class SmoothL1LossKernel : public framework::OpKernel { auto outside_weight = EigenVector::Flatten(*in3); errors.device(place) = errors * outside_weight; } - auto loss = EigenMatrix::From(*out1, {in0->dims()[0], 1}); + auto loss = EigenVector::Flatten(*out1); // first dimension of 'X' is the number of samples - auto errors_mat_view = EigenMatrix::From(paddle_errors, in0->dims()); - loss.device(place) = errors_mat_view.sum(Eigen::array({1})); + auto mat_dims = + framework::make_ddim({static_cast(in0->dims()[0]), + static_cast(in_counts / in0->dims()[0])}); + auto errors_mat_view = EigenMatrix::From(paddle_errors, mat_dims); + loss.device(place) = errors_mat_view.sum(Eigen::array({{1}})); } }; @@ -162,9 +165,9 @@ class SmoothL1LossGradKernel : public framework::OpKernel { // compute gradients auto out_grad = EigenMatrix::From(*og); auto diff_mat_view = EigenMatrix::From(paddle_diff, mat_dims); - auto gradients = - out_grad.broadcast(Eigen::array({1, static_cast(cols)})) * - weights * diff_mat_view; + auto gradients = out_grad.broadcast( + Eigen::array({{1, static_cast(cols)}})) * + weights * diff_mat_view; if (out0) { out0->mutable_data(context.GetPlace()); From f5807670a7d4a00ae95b0fb566dee6ccf39da7cd Mon Sep 17 00:00:00 2001 From: yangyaming Date: Thu, 7 Sep 2017 11:21:25 +0800 Subject: [PATCH 03/35] Fix typos and use HOSTDEVICE instead. --- paddle/operators/smooth_l1_loss_op.h | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/paddle/operators/smooth_l1_loss_op.h b/paddle/operators/smooth_l1_loss_op.h index 3e47403858..bb823a56a3 100644 --- a/paddle/operators/smooth_l1_loss_op.h +++ b/paddle/operators/smooth_l1_loss_op.h @@ -15,6 +15,7 @@ #pragma once #include "paddle/framework/eigen.h" #include "paddle/framework/op_registry.h" +#include "paddle/platform/hostdevice.h" namespace paddle { namespace operators { @@ -28,10 +29,10 @@ template ; template -struct SmoothL1LossFoward { - __host__ __device__ SmoothL1LossFoward(const T& sigma2) : sigma2(sigma2) {} +struct SmoothL1LossForward { + HOSTDEVICE SmoothL1LossForward(const T& sigma2) : sigma2(sigma2) {} - __host__ __device__ T operator()(const T& val) const { + HOSTDEVICE T operator()(const T& val) const { T abs_val = std::abs(val); if (abs_val < 1.0 / sigma2) { return 0.5 * val * val * sigma2; @@ -80,7 +81,7 @@ class SmoothL1LossKernel : public framework::OpKernel { context.GetPlace()); auto errors = EigenVector::Flatten(paddle_errors); // apply smooth l1 forward - errors.device(place) = diff.unaryExpr(SmoothL1LossFoward(sigma2)); + errors.device(place) = diff.unaryExpr(SmoothL1LossForward(sigma2)); // multiply outside weight if (has_weight) { @@ -99,9 +100,9 @@ class SmoothL1LossKernel : public framework::OpKernel { template struct SmoothL1LossBackward { - __host__ __device__ SmoothL1LossBackward(const T& sigma2) : sigma2(sigma2) {} + HOSTDEVICE SmoothL1LossBackward(const T& sigma2) : sigma2(sigma2) {} - __host__ __device__ T operator()(const T& val) const { + HOSTDEVICE T operator()(const T& val) const { T abs_val = std::abs(val); if (abs_val < 1.0 / sigma2) { return sigma2 * val; From 53ab7e78b164a5e1ed8e15aa29ddbcfa5445f338 Mon Sep 17 00:00:00 2001 From: yangyaming Date: Thu, 7 Sep 2017 12:18:40 +0800 Subject: [PATCH 04/35] Adapt new interface. --- paddle/operators/smooth_l1_loss_op.cc | 3 ++- paddle/operators/smooth_l1_loss_op.h | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/paddle/operators/smooth_l1_loss_op.cc b/paddle/operators/smooth_l1_loss_op.cc index e9a3847417..d2c6d955a7 100644 --- a/paddle/operators/smooth_l1_loss_op.cc +++ b/paddle/operators/smooth_l1_loss_op.cc @@ -111,7 +111,8 @@ class SmoothL1LossGradOp : public framework::OperatorWithKernel { namespace ops = paddle::operators; REGISTER_OP(smooth_l1_loss, ops::SmoothL1LossOp, - ops::SmoothL1LossOpMaker, ops::SmoothL1LossGradOp); + ops::SmoothL1LossOpMaker, smooth_l1_loss_grad, + ops::SmoothL1LossGradOp); REGISTER_OP_CPU_KERNEL( smooth_l1_loss, ops::SmoothL1LossKernel); REGISTER_OP_CPU_KERNEL( diff --git a/paddle/operators/smooth_l1_loss_op.h b/paddle/operators/smooth_l1_loss_op.h index bb823a56a3..218fb4c5a5 100644 --- a/paddle/operators/smooth_l1_loss_op.h +++ b/paddle/operators/smooth_l1_loss_op.h @@ -59,7 +59,7 @@ class SmoothL1LossKernel : public framework::OpKernel { out1->mutable_data(context.GetPlace()); auto place = context.GetEigenDevice(); - auto sigma = static_cast(context.op_.GetAttr("sigma")); + auto sigma = static_cast(context.op().Attr("sigma")); T sigma2 = sigma * sigma; bool has_weight = (in2 != nullptr) && (in3 != nullptr); @@ -122,7 +122,7 @@ class SmoothL1LossGradKernel : public framework::OpKernel { auto* in1 = context.Input("OutsideWeight"); auto* in2 = context.Input("diff"); auto* og = context.Input(framework::GradVarName("Out")); - auto sigma = static_cast(context.op_.GetAttr("sigma")); + auto sigma = static_cast(context.op().Attr("sigma")); T sigma2 = sigma * sigma; bool has_weight = (in0 != nullptr) && (in1 != nullptr); From 3a49bae0b465816083861ea58e97a11706fec0c3 Mon Sep 17 00:00:00 2001 From: yangyaming Date: Fri, 8 Sep 2017 17:07:14 +0800 Subject: [PATCH 05/35] Finish forward for GPU and CPU and CPU backward. --- paddle/operators/modified_huber_loss_op.cc | 99 ++++++++++++++ paddle/operators/modified_huber_loss_op.cu | 41 ++++++ paddle/operators/modified_huber_loss_op.h | 126 ++++++++++++++++++ paddle/pybind/pybind.cc | 1 + .../paddle/v2/framework/tests/CMakeLists.txt | 1 + 5 files changed, 268 insertions(+) create mode 100644 paddle/operators/modified_huber_loss_op.cc create mode 100644 paddle/operators/modified_huber_loss_op.cu create mode 100644 paddle/operators/modified_huber_loss_op.h diff --git a/paddle/operators/modified_huber_loss_op.cc b/paddle/operators/modified_huber_loss_op.cc new file mode 100644 index 0000000000..631464bc84 --- /dev/null +++ b/paddle/operators/modified_huber_loss_op.cc @@ -0,0 +1,99 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/modified_huber_loss_op.h" + +namespace paddle { +namespace operators { + +class ModifiedHuberLossOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(const framework::InferShapeContext& context) const override { + PADDLE_ENFORCE_NOT_NULL(context.InputVar("X"), "X must be initialized."); + PADDLE_ENFORCE_NOT_NULL(context.InputVar("Y"), "Y must be initialized."); + + auto* x = context.Input("X"); + auto* y = context.Input("Y"); + + PADDLE_ENFORCE_EQ(x->dims(), y->dims(), + "Dimensions of X and Y must be the same."); + PADDLE_ENFORCE_EQ(framework::arity(x->dims()), 2, + "Tensor rank of X must be 2."); + PADDLE_ENFORCE_EQ(x->dims()[1], 1, "Second dimension of X must be 1."); + + context.Output("intermediate_val")->Resize(x->dims()); + context.Output("Out")->Resize({x->dims()[0], 1}); + } +}; + +class ModifiedHuberLossOpMaker : public framework::OpProtoAndCheckerMaker { + public: + ModifiedHuberLossOpMaker(framework::OpProto* proto, + framework::OpAttrChecker* op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", ""); + AddInput("Y", ""); + AddOutput("intermediate_val", "").AsIntermediate(); + AddOutput("Out", ""); + AddComment(""); + } +}; + +class ModifiedHuberLossGradOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(const framework::InferShapeContext& context) const override { + auto* x = context.Input("X"); + auto* y = context.Input("Y"); + auto* intermediate_val = context.Input("intermediate_val"); + auto* out_grad = context.Input(framework::GradVarName("Out")); + auto* x_grad = context.Output(framework::GradVarName("X")); + auto* y_grad = context.Output(framework::GradVarName("Y")); + + PADDLE_ENFORCE_NOT_NULL(x, "Input X must not be null."); + PADDLE_ENFORCE_NOT_NULL(y, "Target Y must not be null."); + PADDLE_ENFORCE_NOT_NULL(intermediate_val, + "Intermediate value must not be null."); + PADDLE_ENFORCE_NOT_NULL(out_grad, "Out gradient must not be null."); + + PADDLE_ENFORCE_EQ( + intermediate_val->dims(), x->dims(), + "Dimension of X and intermediate value must be the same."); + PADDLE_ENFORCE_EQ( + out_grad->dims(), x->dims(), + "Dimension of Out gradient and X must be the same (N*1)."); + + if (x_grad) x_grad->Resize(x->dims()); + if (y_grad) y_grad->Resize(y->dims()); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP(modified_huber_loss, ops::ModifiedHuberLossOp, + ops::ModifiedHuberLossOpMaker, modified_huber_loss_grad, + ops::ModifiedHuberLossGradOp); + +REGISTER_OP_CPU_KERNEL( + modified_huber_loss, + ops::ModifiedHuberLossKernel); +REGISTER_OP_CPU_KERNEL(modified_huber_loss_grad, + ops::ModifiedHuberLossGradCPUKernel); diff --git a/paddle/operators/modified_huber_loss_op.cu b/paddle/operators/modified_huber_loss_op.cu new file mode 100644 index 0000000000..06c710e0c5 --- /dev/null +++ b/paddle/operators/modified_huber_loss_op.cu @@ -0,0 +1,41 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/framework/op_registry.h" +#include "paddle/operators/modified_huber_loss_op.h" + +namespace paddle { +namespace operators { + +using Tensor = framework::Tensor; + +template +class ModifiedHuberLossGradGPUKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + // auto* in0 = context.Input("X"); + // auto* in1 = context.Input("Y"); + // auto* in2 = context.Input("intermediate_val"); + // auto* in3 = context.Input(framework::GradVarName("Out")); + // auto* out0 = context.Output(framework::GradVarName("X")); + // auto* out1 = context.Output(framework::GradVarName("X")); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP_GPU_KERNEL( + modified_huber_loss, + ops::ModifiedHuberLossKernel); +REGISTER_OP_GPU_KERNEL(modified_huber_loss_grad, + ops::ModifiedHuberLossGradGPUKernel); diff --git a/paddle/operators/modified_huber_loss_op.h b/paddle/operators/modified_huber_loss_op.h new file mode 100644 index 0000000000..2a429ab2e4 --- /dev/null +++ b/paddle/operators/modified_huber_loss_op.h @@ -0,0 +1,126 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once + +#include "paddle/framework/eigen.h" +#include "paddle/framework/op_registry.h" +#include "paddle/platform/hostdevice.h" + +namespace paddle { +namespace operators { + +using Tensor = framework::Tensor; +template +using EigenMatrix = framework::EigenMatrix; +template +using EigenVector = framework::EigenVector; + +template +struct CheckLabelValue { + HOSTDEVICE T operator()(const T& val) const { + PADDLE_ASSERT(val == static_cast(0) || val == static_cast(1)); + } +}; + +template +struct ModifiedHuberLossForward { + HOSTDEVICE T operator()(const T& val) const { + if (val < -1) { + return -4 * val; + } else if (val < 1) { + return (1 - val) * (1 - val); + } else { + return static_cast(0); + } + } +}; + +template +class ModifiedHuberLossKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* in0 = context.Input("X"); + auto* in1 = context.Input("Y"); + auto* out0 = context.Output("intermediate_val"); + auto* out1 = context.Output("Out"); + + out0->mutable_data(context.GetPlace()); + out1->mutable_data(context.GetPlace()); + auto place = context.GetEigenDevice(); + + auto x = EigenVector::Flatten(*in0); + auto y = EigenVector::Flatten(*in1); + // make sure value's of Y in {0, 1} + y.unaryExpr(CheckLabelValue()); + auto inter_val = EigenVector::Flatten(*out0); + // scale y to {-1, +1} and compute x * y + inter_val.device(place) = x * (2 * y - static_cast(1)); + auto loss = EigenVector::Flatten(*out1); + loss.device(place) = inter_val.unaryExpr(ModifiedHuberLossForward()); + } +}; + +// Use thrust lib to unify cpu and gpu +// CPU backward kernel +template +class ModifiedHuberLossGradCPUKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + auto* in0 = context.Input("X"); + auto* in1 = context.Input("Y"); + auto* in2 = context.Input("intermediate_val"); + auto* in3 = context.Input(framework::GradVarName("Out")); + auto* out0 = context.Output(framework::GradVarName("X")); + auto* out1 = context.Output(framework::GradVarName("X")); + + // loop inter_val (x<-1) (x<1) otherwise + const T* p_inter_val = in2->data(); + const T* p_out_grad = in3->data(); + size_t counts = static_cast(framework::product(in2->dims())); + + if (out0) { + T* p_x_grad = out0->mutable_data(context.GetPlace()); + const T* p_y = in1->data(); + ModifiedHuberLossBackward(p_inter_val, p_y, p_out_grad, p_x_grad, counts); + } + + if (out1) { + T* p_y_grad = out1->mutable_data(context.GetPlace()); + const T* p_x = in0->data(); + ModifiedHuberLossBackward(p_inter_val, p_x, p_out_grad, p_y_grad, counts); + } + } + + protected: + void ModifiedHuberLossBackward(const T* p_inter_data, const T* p_in_data, + const T* p_in_grad, T* p_out_grad, + size_t counts) const { + for (size_t i = 0; i < counts; ++i) { + if (p_inter_data[i] < -1) { + p_out_grad[i] = -4 * p_in_data[i] * p_in_grad[i]; + } else if (p_inter_data[i] < 1) { + p_out_grad[i] = + -2 * (1 - p_inter_data[i]) * p_in_data[i] * p_in_grad[i]; + } else { + p_out_grad[i] = 0; + } + } + } +}; + +} // namespace operators +} // namespace paddle diff --git a/paddle/pybind/pybind.cc b/paddle/pybind/pybind.cc index c21ad3470b..79e02c57b8 100644 --- a/paddle/pybind/pybind.cc +++ b/paddle/pybind/pybind.cc @@ -50,6 +50,7 @@ USE_OP(cos_sim); USE_CPU_ONLY_OP(gather); USE_CPU_ONLY_OP(scatter); USE_OP(squared_l2_distance); +USE_OP(modified_huber_loss); namespace paddle { namespace framework { diff --git a/python/paddle/v2/framework/tests/CMakeLists.txt b/python/paddle/v2/framework/tests/CMakeLists.txt index a9c33ea163..10e1c33962 100644 --- a/python/paddle/v2/framework/tests/CMakeLists.txt +++ b/python/paddle/v2/framework/tests/CMakeLists.txt @@ -34,3 +34,4 @@ py_test(test_lookup_table SRCS test_lookup_table.py) py_test(test_scale_and_identity_op SRCS test_scale_and_identity_op.py) py_test(mnist SRCS mnist.py) py_test(test_squared_l2_distance_op SRCS test_squared_l2_distance_op.py) +py_test(test_modified_huber_loss_op SRCS test_modified_huber_loss_op.py) From 076dcb9b45a1a8dd385eee58aa42da042798cada Mon Sep 17 00:00:00 2001 From: yangyaming Date: Fri, 8 Sep 2017 17:21:17 +0800 Subject: [PATCH 06/35] Simpify the initialization for weights. --- paddle/operators/smooth_l1_loss_op.h | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/paddle/operators/smooth_l1_loss_op.h b/paddle/operators/smooth_l1_loss_op.h index 218fb4c5a5..8af831ae35 100644 --- a/paddle/operators/smooth_l1_loss_op.h +++ b/paddle/operators/smooth_l1_loss_op.h @@ -141,22 +141,12 @@ class SmoothL1LossGradKernel : public framework::OpKernel { diff.device(place) = EigenVector::Flatten(*in2).unaryExpr( SmoothL1LossBackward(sigma2)); - auto* out0 = context.Output(framework::GradVarName("X")); - auto* out1 = context.Output(framework::GradVarName("Y")); - // compute weights Tensor paddle_weights; paddle_weights.mutable_data(mat_dims, context.GetPlace()); auto weights = EigenMatrix::From(paddle_weights); // initialize to 1.0 - if (platform::is_cpu_place(context.GetPlace())) { - weights.setConstant(static_cast(1.0)); - } else { - Tensor paddle_cpu_weights; - paddle_cpu_weights.mutable_data(mat_dims, platform::CPUPlace()); - EigenMatrix::From(paddle_cpu_weights).setConstant(static_cast(1.0)); - paddle_weights.CopyFrom(paddle_cpu_weights, context.GetPlace()); - } + weights.device(place) = weights.constant(static_cast(1.0)); if (has_weight) { auto inside_weight = EigenMatrix::From(*in0, mat_dims); auto outside_weight = EigenMatrix::From(*in1, mat_dims); @@ -170,6 +160,9 @@ class SmoothL1LossGradKernel : public framework::OpKernel { Eigen::array({{1, static_cast(cols)}})) * weights * diff_mat_view; + auto* out0 = context.Output(framework::GradVarName("X")); + auto* out1 = context.Output(framework::GradVarName("Y")); + if (out0) { out0->mutable_data(context.GetPlace()); auto x_grad = EigenMatrix::From(*out0, mat_dims); From 984117458ca019335d4ba8cd111f0895800651aa Mon Sep 17 00:00:00 2001 From: yangyaming Date: Sat, 9 Sep 2017 16:55:19 +0800 Subject: [PATCH 07/35] Finish modified huber loss op. --- paddle/operators/modified_huber_loss_op.cc | 26 ++++++--- paddle/operators/modified_huber_loss_op.cu | 49 +++++++++++++++-- paddle/operators/modified_huber_loss_op.h | 52 ++++++------------ .../tests/test_modified_huber_loss_op.py | 55 +++++++++++++++++++ 4 files changed, 134 insertions(+), 48 deletions(-) create mode 100644 python/paddle/v2/framework/tests/test_modified_huber_loss_op.py diff --git a/paddle/operators/modified_huber_loss_op.cc b/paddle/operators/modified_huber_loss_op.cc index 631464bc84..631d406fd4 100644 --- a/paddle/operators/modified_huber_loss_op.cc +++ b/paddle/operators/modified_huber_loss_op.cc @@ -45,11 +45,25 @@ class ModifiedHuberLossOpMaker : public framework::OpProtoAndCheckerMaker { ModifiedHuberLossOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("X", ""); - AddInput("Y", ""); - AddOutput("intermediate_val", "").AsIntermediate(); - AddOutput("Out", ""); - AddComment(""); + AddInput("X", "Input value of ModifiedHuberLossOp."); + AddInput("Y", "Target labels of ModifiedHuberLossOp."); + AddOutput("intermediate_val", + "Variable to save intermediate result which will be reused in " + "backward processing.") + .AsIntermediate(); + AddOutput("Out", "Classification loss for input X."); + AddComment(R"DOC( +Modified huber loss is used in binary classification problem. Dimensions of +input X and target Y are both (N, 1) and so is the dimension of output loss. +Since target Y is not differentiable, cacluating gradient for Y is illegal. +The formulation of modified huber loss is: + +L(y, f(x)) = max(0, 1 - yf(x))^2 for yf(x) >= -1, + -4yf(x) otherwise. + +Make sure the values of target label Y are in {0, 1} here. The operator will +scale values of Y to {-1, +1} when computing loss and gradients. +)DOC"); } }; @@ -64,7 +78,6 @@ class ModifiedHuberLossGradOp : public framework::OperatorWithKernel { auto* intermediate_val = context.Input("intermediate_val"); auto* out_grad = context.Input(framework::GradVarName("Out")); auto* x_grad = context.Output(framework::GradVarName("X")); - auto* y_grad = context.Output(framework::GradVarName("Y")); PADDLE_ENFORCE_NOT_NULL(x, "Input X must not be null."); PADDLE_ENFORCE_NOT_NULL(y, "Target Y must not be null."); @@ -80,7 +93,6 @@ class ModifiedHuberLossGradOp : public framework::OperatorWithKernel { "Dimension of Out gradient and X must be the same (N*1)."); if (x_grad) x_grad->Resize(x->dims()); - if (y_grad) y_grad->Resize(y->dims()); } }; diff --git a/paddle/operators/modified_huber_loss_op.cu b/paddle/operators/modified_huber_loss_op.cu index 06c710e0c5..f8aa5043dd 100644 --- a/paddle/operators/modified_huber_loss_op.cu +++ b/paddle/operators/modified_huber_loss_op.cu @@ -9,24 +9,61 @@ See the License for the specific language governing permissions and limitations under the License. */ +#include +#include +#include +#include #include "paddle/framework/op_registry.h" #include "paddle/operators/modified_huber_loss_op.h" +#include "paddle/platform/hostdevice.h" namespace paddle { namespace operators { using Tensor = framework::Tensor; +struct ModifiedHuberLossBackward { + template + HOSTDEVICE void operator()(Tuple t) const { + auto inter_val = thrust::get<1>(t); + auto y_val = thrust::get<2>(t); + auto out_grad = thrust::get<3>(t); + if (inter_val < -1) { + thrust::get<0>(t) = -4 * (2 * y_val - 1) * out_grad; + } else if (inter_val < 1) { + thrust::get<0>(t) = -2 * (1 - inter_val) * (2 * y_val - 1) * out_grad; + } else { + thrust::get<0>(t) = 0; + } + } +}; + template class ModifiedHuberLossGradGPUKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { - // auto* in0 = context.Input("X"); - // auto* in1 = context.Input("Y"); - // auto* in2 = context.Input("intermediate_val"); - // auto* in3 = context.Input(framework::GradVarName("Out")); - // auto* out0 = context.Output(framework::GradVarName("X")); - // auto* out1 = context.Output(framework::GradVarName("X")); + auto* in0 = context.Input("Y"); + auto* in1 = context.Input("intermediate_val"); + auto* in2 = context.Input(framework::GradVarName("Out")); + auto* out0 = context.Output(framework::GradVarName("X")); + + if (out0) { + auto counts = framework::product(in1->dims()); + auto y_ptr = thrust::device_pointer_cast(in0->data()); + auto inter_val_ptr = thrust::device_pointer_cast(in1->data()); + auto out_grad_ptr = thrust::device_pointer_cast(in2->data()); + thrust::device_ptr x_grad_ptr( + out0->mutable_data(context.GetPlace())); + + auto iter_begin = thrust::make_zip_iterator( + thrust::make_tuple(x_grad_ptr, inter_val_ptr, y_ptr, out_grad_ptr)); + + auto iter_end = thrust::make_zip_iterator( + thrust::make_tuple(x_grad_ptr + counts, inter_val_ptr + counts, + y_ptr + counts, out_grad_ptr + counts)); + + thrust::for_each(iter_begin, iter_end, ModifiedHuberLossBackward()); + } } }; diff --git a/paddle/operators/modified_huber_loss_op.h b/paddle/operators/modified_huber_loss_op.h index 2a429ab2e4..13c11684af 100644 --- a/paddle/operators/modified_huber_loss_op.h +++ b/paddle/operators/modified_huber_loss_op.h @@ -74,49 +74,31 @@ class ModifiedHuberLossKernel : public framework::OpKernel { } }; -// Use thrust lib to unify cpu and gpu // CPU backward kernel template class ModifiedHuberLossGradCPUKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { - auto* in0 = context.Input("X"); - auto* in1 = context.Input("Y"); - auto* in2 = context.Input("intermediate_val"); - auto* in3 = context.Input(framework::GradVarName("Out")); + auto* in0 = context.Input("Y"); + auto* in1 = context.Input("intermediate_val"); + auto* in2 = context.Input(framework::GradVarName("Out")); auto* out0 = context.Output(framework::GradVarName("X")); - auto* out1 = context.Output(framework::GradVarName("X")); - - // loop inter_val (x<-1) (x<1) otherwise - const T* p_inter_val = in2->data(); - const T* p_out_grad = in3->data(); - size_t counts = static_cast(framework::product(in2->dims())); if (out0) { - T* p_x_grad = out0->mutable_data(context.GetPlace()); - const T* p_y = in1->data(); - ModifiedHuberLossBackward(p_inter_val, p_y, p_out_grad, p_x_grad, counts); - } - - if (out1) { - T* p_y_grad = out1->mutable_data(context.GetPlace()); - const T* p_x = in0->data(); - ModifiedHuberLossBackward(p_inter_val, p_x, p_out_grad, p_y_grad, counts); - } - } - - protected: - void ModifiedHuberLossBackward(const T* p_inter_data, const T* p_in_data, - const T* p_in_grad, T* p_out_grad, - size_t counts) const { - for (size_t i = 0; i < counts; ++i) { - if (p_inter_data[i] < -1) { - p_out_grad[i] = -4 * p_in_data[i] * p_in_grad[i]; - } else if (p_inter_data[i] < 1) { - p_out_grad[i] = - -2 * (1 - p_inter_data[i]) * p_in_data[i] * p_in_grad[i]; - } else { - p_out_grad[i] = 0; + const T* y_ptr = in0->data(); + const T* inter_val_ptr = in1->data(); + const T* out_grad_ptr = in2->data(); + size_t counts = static_cast(framework::product(in1->dims())); + T* x_grad_ptr = out0->mutable_data(context.GetPlace()); + for (size_t i = 0; i < counts; ++i) { + if (inter_val_ptr[i] < -1) { + x_grad_ptr[i] = -4 * (2 * y_ptr[i] - 1) * out_grad_ptr[i]; + } else if (inter_val_ptr[i] < 1) { + x_grad_ptr[i] = -2 * (1 - inter_val_ptr[i]) * (2 * y_ptr[i] - 1) * + out_grad_ptr[i]; + } else { + x_grad_ptr[i] = 0; + } } } } diff --git a/python/paddle/v2/framework/tests/test_modified_huber_loss_op.py b/python/paddle/v2/framework/tests/test_modified_huber_loss_op.py new file mode 100644 index 0000000000..2b76c53b6e --- /dev/null +++ b/python/paddle/v2/framework/tests/test_modified_huber_loss_op.py @@ -0,0 +1,55 @@ +import unittest +from op_test_util import OpTestMeta +from gradient_checker import GradientChecker, create_op +from paddle.v2.framework.op import Operator +import numpy as np + + +def modified_huber_loss_forward(val): + if val < -1: + return -4 * a + elif val < 1: + return (1 - val) * (1 - val) + else: + return 0 + + +class TestModifiedHuberLossOp_f0(unittest.TestCase): + __metaclass__ = OpTestMeta + + def setUp(self): + self.type = 'modified_huber_loss' + samples_num = 32 + self.inputs = { + 'X': np.random.uniform(-1, 1., (samples_num, 1)).astype('float32'), + 'Y': np.random.choice([0, 1], samples_num).reshape((samples_num, 1)) + } + product_res = self.inputs['X'] * (2 * self.inputs['Y'] - 1) + loss = np.vectorize(modified_huber_loss_forward)(product_res) + + self.outputs = { + 'intermediate_val': product_res, + 'Out': loss.reshape((samples_num, 1)) + } + + +class TestModifiedHuberLossGradOp(GradientChecker): + def test_modified_huber_loss_b0(self): + samples_num = 10 + inputs = { + 'X': np.random.uniform(-1, 1, (samples_num, 1)).astype('float32'), + 'Y': np.random.choice([0, 1], samples_num).reshape((samples_num, 1)) + } + op = Operator( + "modified_huber_loss", + X='X', + Y='Y', + intermediate_val='intermediate_val', + Out='Out') + self.compare_grad( + op, inputs, no_grad_set=set(['intermediate_val', 'Y'])) + self.check_grad(op, inputs, set(["X"]), "Out") + + +if __name__ == '__main__': + unittest.main() From 696b1f5fa97d20102ab8947c4e01e588fd4926ad Mon Sep 17 00:00:00 2001 From: yangyaming Date: Sat, 9 Sep 2017 17:06:37 +0800 Subject: [PATCH 08/35] Refine doc. --- paddle/operators/smooth_l1_loss_op.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/paddle/operators/smooth_l1_loss_op.cc b/paddle/operators/smooth_l1_loss_op.cc index d2c6d955a7..5c704c35b4 100644 --- a/paddle/operators/smooth_l1_loss_op.cc +++ b/paddle/operators/smooth_l1_loss_op.cc @@ -70,14 +70,14 @@ class SmoothL1LossOpMaker : public framework::OpProtoAndCheckerMaker { AddOutput("diff", "Intermediate variable to cache Win*(X-Y).") .AsIntermediate(); AddOutput("Out", "Final smooth l1 loss of inputs."); + AddAttr("sigma", "Hyper parameter, default value is 3.0 .") + .SetDefault(3.0); AddComment(R"DOC( Compute SmoothL1Loss for input and target. - -The equation is: Out = 0.5 * (sigma * (X - Y)) ^ 2 if abs(X - Y) < 1 / sigma^2 - abs(X - Y) - 0.5 / sigma^2 otherwise +The equation is: +loss = 0.5 * (sigma * (x - y)) ^ 2 if abs(x - y) < 1 / sigma^2 + abs(x - y) - 0.5 / sigma^2 otherwise )DOC"); - AddAttr("sigma", "Hyper parameter, default value is 3.0 .") - .SetDefault(3.0); } }; From 36e50135b8f63fe1d55c63f39b84756757ba5392 Mon Sep 17 00:00:00 2001 From: yangyaming Date: Sat, 9 Sep 2017 19:05:03 +0800 Subject: [PATCH 09/35] Remove EigenMatrix define. --- paddle/operators/modified_huber_loss_op.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/paddle/operators/modified_huber_loss_op.h b/paddle/operators/modified_huber_loss_op.h index 13c11684af..ffb89e806f 100644 --- a/paddle/operators/modified_huber_loss_op.h +++ b/paddle/operators/modified_huber_loss_op.h @@ -22,9 +22,6 @@ namespace paddle { namespace operators { using Tensor = framework::Tensor; -template -using EigenMatrix = framework::EigenMatrix; template using EigenVector = framework::EigenVector; From ce15d89afa3caa372f25b315bafcec1a1dfcd82c Mon Sep 17 00:00:00 2001 From: yangyaming Date: Wed, 13 Sep 2017 21:55:18 +0800 Subject: [PATCH 10/35] Adapt to new unittest. --- .../tests/test_modified_huber_loss_op.py | 29 +++++-------------- 1 file changed, 7 insertions(+), 22 deletions(-) diff --git a/python/paddle/v2/framework/tests/test_modified_huber_loss_op.py b/python/paddle/v2/framework/tests/test_modified_huber_loss_op.py index 64ac363ce0..a7e2b57529 100644 --- a/python/paddle/v2/framework/tests/test_modified_huber_loss_op.py +++ b/python/paddle/v2/framework/tests/test_modified_huber_loss_op.py @@ -1,8 +1,6 @@ import unittest -from op_test_util import OpTestMeta -from gradient_checker import GradientChecker, create_op -from paddle.v2.framework.op import Operator import numpy as np +from op_test import OpTest def modified_huber_loss_forward(val): @@ -14,11 +12,9 @@ def modified_huber_loss_forward(val): return 0 -class TestModifiedHuberLossOp_f0(unittest.TestCase): - __metaclass__ = OpTestMeta - +class TestModifiedHuberLossOp(OpTest): def setUp(self): - self.type = 'modified_huber_loss' + self.op_type = 'modified_huber_loss' samples_num = 32 self.inputs = { 'X': np.random.uniform(-1, 1., (samples_num, 1)).astype('float32'), @@ -32,22 +28,11 @@ class TestModifiedHuberLossOp_f0(unittest.TestCase): 'Out': loss.reshape((samples_num, 1)) } + def test_check_output(self): + self.check_output() -class TestModifiedHuberLossGradOp(GradientChecker): - def test_modified_huber_loss_b0(self): - samples_num = 10 - inputs = { - 'X': np.random.uniform(-1, 1, (samples_num, 1)).astype('float32'), - 'Y': np.random.choice([0, 1], samples_num).reshape((samples_num, 1)) - } - op = Operator( - "modified_huber_loss", - X='X', - Y='Y', - IntermediateVal='IntermediateVal', - Out='Out') - self.compare_grad(op, inputs, no_grad_set=set(['IntermediateVal', 'Y'])) - self.check_grad(op, inputs, set(["X"]), "Out") + def test_check_grad(self): + self.check_grad(['X'], 'Out', max_relative_error=0.005) if __name__ == '__main__': From 3ee87653b451f805e2f153d8a872846fc0b42f63 Mon Sep 17 00:00:00 2001 From: yangyaming Date: Fri, 15 Sep 2017 17:31:17 +0800 Subject: [PATCH 11/35] Tight the relative error. --- .../v2/framework/tests/test_smooth_l1_loss_op.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/python/paddle/v2/framework/tests/test_smooth_l1_loss_op.py b/python/paddle/v2/framework/tests/test_smooth_l1_loss_op.py index 3ab7c6bb29..1b79f16abe 100644 --- a/python/paddle/v2/framework/tests/test_smooth_l1_loss_op.py +++ b/python/paddle/v2/framework/tests/test_smooth_l1_loss_op.py @@ -14,7 +14,7 @@ def smooth_l1_loss_forward(val, sigma2): class TestSmoothL1LossOp1(OpTest): def setUp(self): self.op_type = "smooth_l1_loss" - dims = (10, 15) + dims = (6, 10) self.inputs = { 'X': np.random.random(dims).astype("float32"), 'Y': np.random.random(dims).astype("float32") @@ -31,21 +31,21 @@ class TestSmoothL1LossOp1(OpTest): self.check_output() def test_check_grad_normal(self): - self.check_grad(['X', 'Y'], 'Out', max_relative_error=0.08) + self.check_grad(['X', 'Y'], 'Out', max_relative_error=0.02) def test_check_grad_ingore_x(self): self.check_grad( - ['Y'], 'Out', max_relative_error=0.08, no_grad_set=set("X")) + ['Y'], 'Out', max_relative_error=0.02, no_grad_set=set("X")) def test_check_grad_ingore_y(self): self.check_grad( - ['X'], 'Out', max_relative_error=0.08, no_grad_set=set('Y')) + ['X'], 'Out', max_relative_error=0.02, no_grad_set=set('Y')) class TestSmoothL1LossOp2(OpTest): def setUp(self): self.op_type = "smooth_l1_loss" - dims = (10, 15) + dims = (6, 10) self.inputs = { 'X': np.random.random(dims).astype("float32"), 'Y': np.random.random(dims).astype("float32"), @@ -66,20 +66,20 @@ class TestSmoothL1LossOp2(OpTest): self.check_output() def test_check_grad_normal(self): - self.check_grad(['X', 'Y'], 'Out', max_relative_error=0.08) + self.check_grad(['X', 'Y'], 'Out', max_relative_error=0.02) def test_check_grad_ingore_x(self): self.check_grad( ['Y'], 'Out', - max_relative_error=0.08, + max_relative_error=0.02, no_grad_set=set(['X', 'InsideWeight', 'OutsideWeight'])) def test_check_grad_ingore_y(self): self.check_grad( ['X'], 'Out', - max_relative_error=0.08, + max_relative_error=0.02, no_grad_set=set(['Y', 'InsideWeight', 'OutsideWeight'])) From c2dea5a877611f43ea8f7100297305297fa19ef9 Mon Sep 17 00:00:00 2001 From: ranqiu Date: Mon, 18 Sep 2017 17:14:38 +0800 Subject: [PATCH 12/35] Update the annotation of layers.py --- .../paddle/trainer_config_helpers/layers.py | 67 +++++++++++-------- 1 file changed, 40 insertions(+), 27 deletions(-) diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index 4b1d80d3db..d4d165e5d9 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -5457,9 +5457,9 @@ def nce_layer(input, param_attr=[attr1, attr2], weight=layer3, num_classes=3, neg_distribution=[0.1,0.3,0.6]) - :param name: layer name + :param name: The name of this layer. :type name: basestring - :param input: input layers. It could be a LayerOutput of list/tuple of LayerOutput. + :param input: The input layers. It could be a LayerOutput of list/tuple of LayerOutput. :type input: LayerOutput|list|tuple|collections.Sequence :param label: label layer :type label: LayerOutput @@ -5477,7 +5477,9 @@ def nce_layer(input, A uniform distribution will be used if not provided. If not None, its length must be equal to num_classes. :type neg_distribution: list|tuple|collections.Sequence|None - :param bias_attr: Bias parameter attribute. True if no bias. + :param bias_attr: The Bias Attribute. If no bias, then pass False or + something not type of ParameterAttribute. None will get a + default Bias. :type bias_attr: ParameterAttribute|None|False :param layer_attr: Extra Layer Attribute. :type layer_attr: ExtraLayerAttribute @@ -5593,7 +5595,7 @@ def rank_cost(left, :param weight: The weight affects the cost, namely the scale of cost. It is an optional argument. :type weight: LayerOutput - :param name: The name of this layers. It is not necessary. + :param name: The name of this layer. It is not necessary. :type name: None|basestring :param coeff: The coefficient affects the gradient in the backward. :type coeff: float @@ -5647,7 +5649,7 @@ def lambda_cost(input, :param score: The 2nd input. Score of each sample. :type input: LayerOutput :param NDCG_num: The size of NDCG (Normalized Discounted Cumulative Gain), - e.g., 5 for NDCG@5. It must be less than for equal to the + e.g., 5 for NDCG@5. It must be less than or equal to the minimum size of lists. :type NDCG_num: int :param max_sort_size: The size of partial sorting in calculating gradient. @@ -5658,7 +5660,7 @@ def lambda_cost(input, than the size of a list, the algorithm will sort the entire list of get gradient. :type max_sort_size: int - :param name: The name of this layers. It is not necessary. + :param name: The name of this layer. It is not necessary. :type name: None|basestring :param layer_attr: Extra Layer Attribute. :type layer_attr: ExtraLayerAttribute @@ -5702,7 +5704,7 @@ def cross_entropy(input, :type input: LayerOutput. :param label: The input label. :type input: LayerOutput. - :param name: The name of this layers. It is not necessary. + :param name: The name of this layer. It is not necessary. :type name: None|basestring. :param coeff: The cost is multiplied with coeff. The coefficient affects the gradient in the backward. @@ -5750,7 +5752,7 @@ def cross_entropy_with_selfnorm(input, :type input: LayerOutput. :param label: The input label. :type input: LayerOutput. - :param name: The name of this layers. It is not necessary. + :param name: The name of this layer. It is not necessary. :type name: None|basestring. :param coeff: The coefficient affects the gradient in the backward. :type coeff: float. @@ -5790,7 +5792,7 @@ def sum_cost(input, name=None, layer_attr=None): :param input: The first input layer. :type input: LayerOutput. - :param name: The name of this layers. It is not necessary. + :param name: The name of this layer. It is not necessary. :type name: None|basestring. :param layer_attr: Extra Layer Attribute. :type layer_attr: ExtraLayerAttribute @@ -5835,7 +5837,7 @@ def huber_regression_cost(input, :type input: LayerOutput. :param label: The input label. :type input: LayerOutput. - :param name: The name of this layers. It is not necessary. + :param name: The name of this layer. It is not necessary. :type name: None|basestring. :param delta: The difference between the observed and predicted values. :type delta: float. @@ -5885,7 +5887,7 @@ def huber_classification_cost(input, :type input: LayerOutput. :param label: The input label. :type input: LayerOutput. - :param name: The name of this layers. It is not necessary. + :param name: The name of this layer. It is not necessary. :type name: None|basestring. :param coeff: The coefficient affects the gradient in the backward. :type coeff: float. @@ -5928,7 +5930,7 @@ def multi_binary_label_cross_entropy(input, :type input: LayerOutput :param label: The input label. :type input: LayerOutput - :param name: The name of this layers. It is not necessary. + :param name: The name of this layer. It is not necessary. :type name: None|basestring :param coeff: The coefficient affects the gradient in the backward. :type coeff: float @@ -6033,9 +6035,9 @@ def cross_entropy_over_beam(input, name=None): ]) - :param input: input beams for this layer. + :param input: Input beams for this layer. :type input: BeamInput - :param name: input beams for this layer. + :param name: The name of this layer. :type name: basestring :return: LayerOutput object. :rtype: LayerOutput @@ -6175,12 +6177,21 @@ def multiplex_layer(input, name=None, layer_attr=None): @wrap_name_default("dropout") def dropout_layer(input, dropout_rate, name=None): """ - @TODO(yuyang18): Add comments. - :param name: - :param input: - :param dropout_rate: - :return: + The example usage is: + + .. code-block:: python + + dropout = dropout_layer(input=input_layer, dropout_rate=0.5) + + :param name: The name of this layer. + :type name: basestring + :param input: The input layer. + :type input: LayerOutput + :param dropout_rate: The probability of dropout. + :type dropout_rate: float + :return: LayerOutput object. + :rtype: LayerOutput """ return addto_layer( name=name, @@ -6203,7 +6214,7 @@ def row_conv_layer(input, """ The row convolution is called lookahead convolution. It is firstly - introduced in paper of `Deep Speech 2: End-toEnd Speech Recognition + introduced in paper of `Deep Speech 2: End-to-End Speech Recognition in English and Mandarin `_ . The bidirectional RNN that learns representation for a sequence by @@ -6211,9 +6222,9 @@ def row_conv_layer(input, However, unlike unidirectional RNNs, bidirectional RNNs are challenging to deploy in an online and low-latency setting. The lookahead convolution incorporates information from future subsequences in a computationally - efficient manner to improve unidirectional recurrent neural networks. + efficient manner to improve unidirectional RNNs. - The connection of row convolution is different form the 1D sequence + The connection of row convolution is different from the 1D sequence convolution. Assumed that, the future context-length is k, that is to say, it can get the output at timestep t by using the the input feature from t-th timestep to (t+k+1)-th timestep. Assumed that the hidden dim of input @@ -6242,7 +6253,7 @@ def row_conv_layer(input, :param act: Activation Type. Default is linear activation. :type act: BaseActivation :param param_attr: The Parameter Attribute. If None, the parameter will be - initialized smartly. It's better set it by yourself. + initialized smartly. It's better to set it by yourself. :type param_attr: ParameterAttribute :param layer_attr: Extra Layer config. :type layer_attr: ExtraLayerAttribute|None @@ -6342,7 +6353,7 @@ def gated_unit_layer(input, The gated unit layer implements a simple gating mechanism over the input. The input :math:`X` is first projected into a new space :math:`X'`, and it is also used to produce a gate weight :math:`\sigma`. Element-wise - prodict between :match:`X'` and :math:`\sigma` is finally returned. + product between :match:`X'` and :math:`\sigma` is finally returned. Reference: Language Modeling with Gated Convolutional Networks @@ -6440,8 +6451,8 @@ def switch_order_layer(input, :type input: LayerOutput :param name: Name of this layer. :type name: basestring - :param reshape: reshape matrix by axises. - :type reshape: Dict + :param reshape_axis: Specify the axises of 'height'. Its value should be positive and less than 4. + :type reshape_axis: int :return: LayerOutput object. :rtype: LayerOutput """ @@ -6869,7 +6880,9 @@ def scale_shift_layer(input, name=None, param_attr=None, bias_attr=None): :type input: LayerOutput. :param param_attr: The parameter attribute of scaling. :type param_attr: ParameterAttribute - :param bias_attr: The parameter attribute of shifting. + :param bias_attr: The Bias Attribute. If no bias, then pass False or + something not type of ParameterAttribute. None will get a + default Bias. :type bias_attr: ParameterAttribute :return: LayerOutput object. :rtype: LayerOutput From a0187f1c55e2522d5a8357027888a9870126a81f Mon Sep 17 00:00:00 2001 From: ranqiu Date: Mon, 18 Sep 2017 21:08:16 +0800 Subject: [PATCH 13/35] Update the annotation about bias_attr of layers.py --- .../paddle/trainer_config_helpers/layers.py | 168 +++++++++++------- 1 file changed, 102 insertions(+), 66 deletions(-) diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index 0983d2c3e6..a6fbff9ca4 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -781,11 +781,11 @@ class MixedLayerType(LayerOutput): :type size: int :param act: activation type. :type act: BaseActivation - :param bias_attr: The Bias Attribute. If no bias, then pass False or - something not type of ParameterAttribute. None will - get a default Bias. - :type bias_attr: ParameterAttribute or None means has bias. Any other - type means no bias. + :param bias_attr: The Bias Attribute. If the parameter is set to + False or something not type of ParameterAttribute, + no bias is defined. If the parameter is set to + True, the bias is initialized to zero. + :type bias_attr: ParameterAttribute|None|Bool|Any :param layer_attr: Extra Layer Attribute. :type layer_attr: ExtraLayerAttribute or None """ @@ -881,10 +881,11 @@ def mixed_layer(size=0, then this function will just return layer's name. :param act: Activation Type. :type act: BaseActivation - :param bias_attr: The Bias Attribute. If no bias, then pass False or - something not type of ParameterAttribute. None will get a - default Bias. - :type bias_attr: ParameterAttribute or None or bool + :param bias_attr: The Bias Attribute. If the parameter is set to + False or something not type of ParameterAttribute, + no bias is defined. If the parameter is set to + True, the bias is initialized to zero. + :type bias_attr: ParameterAttribute|None|Bool|Any :param layer_attr: The extra layer config. Default is None. :type layer_attr: ExtraLayerAttribute :return: MixedLayerType object can add inputs or layer name. @@ -1025,10 +1026,11 @@ def fc_layer(input, :type act: BaseActivation :param param_attr: The Parameter Attribute|list. :type param_attr: ParameterAttribute - :param bias_attr: The Bias Attribute. If no bias, then pass False or - something not type of ParameterAttribute. None will get a - default Bias. - :type bias_attr: ParameterAttribute|None|Any + :param bias_attr: The Bias Attribute. If the parameter is set to + False or something not type of ParameterAttribute, + no bias is defined. If the parameter is set to + True, the bias is initialized to zero. + :type bias_attr: ParameterAttribute|None|Bool|Any :param layer_attr: Extra Layer config. :type layer_attr: ExtraLayerAttribute|None :return: LayerOutput object. @@ -1373,8 +1375,11 @@ def pooling_layer(input, :type pooling_type: BasePoolingType|None :param stride: The step size between successive pooling regions. :type stride: Int - :param bias_attr: Bias parameter attribute. False if no bias. - :type bias_attr: ParameterAttribute|None|False + :param bias_attr: The Bias Attribute. If the parameter is set to + False or something not type of ParameterAttribute, + no bias is defined. If the parameter is set to + True, the bias is initialized to zero. + :type bias_attr: ParameterAttribute|None|Bool|Any :param layer_attr: The Extra Attributes for layer, such as dropout. :type layer_attr: ExtraLayerAttribute|None :return: LayerOutput object. @@ -1471,10 +1476,11 @@ def lstmemory(input, :type gate_act: BaseActivation :param state_act: state activation type, TanhActivation by default. :type state_act: BaseActivation - - :param bias_attr: Bias attribute. None means default bias. False means no - bias. - :type bias_attr: ParameterAttribute|None|False + :param bias_attr: The Bias Attribute. If the parameter is set to + False or something not type of ParameterAttribute, + no bias is defined. If the parameter is set to + True, the bias is initialized to zero. + :type bias_attr: ParameterAttribute|None|Bool|Any :param param_attr: Parameter Attribute. :type param_attr: ParameterAttribute|None|False :param layer_attr: Extra Layer attribute @@ -1596,9 +1602,11 @@ def grumemory(input, This activation affects the :math:`z_t` and :math:`r_t`. It is the :math:`\\sigma` in the above formula. :type gate_act: BaseActivation - :param bias_attr: Bias attribute. None means default bias. False means no - bias. - :type bias_attr: ParameterAttribute|None|False + :param bias_attr: The Bias Attribute. If the parameter is set to + False or something not type of ParameterAttribute, + no bias is defined. If the parameter is set to + True, the bias is initialized to zero. + :type bias_attr: ParameterAttribute|None|Bool|Any :param param_attr: Parameter Attribute. :type param_attr: ParameterAttribute|None|False :param layer_attr: Extra Layer attribute @@ -1794,9 +1802,11 @@ def expand_layer(input, :type expand_as: LayerOutput :param name: Layer name. :type name: basestring - :param bias_attr: Bias attribute. None means default bias. False means no - bias. - :type bias_attr: ParameterAttribute|None|False + :param bias_attr: The Bias Attribute. If the parameter is set to + False or something not type of ParameterAttribute, + no bias is defined. If the parameter is set to + True, the bias is initialized to zero. + :type bias_attr: ParameterAttribute|None|Bool|Any :param expand_level: whether input layer is timestep(default) or sequence. :type expand_level: ExpandLevel :param layer_attr: extra layer attributes. @@ -1914,10 +1924,11 @@ def seq_reshape_layer(input, :type act: BaseActivation :param layer_attr: extra layer attributes. :type layer_attr: ExtraLayerAttribute. - :param bias_attr: The Bias Attribute. If no bias, then pass False or - something not type of ParameterAttribute. None will get a - default Bias. - :type bias_attr: ParameterAttribute or None or bool + :param bias_attr: The Bias Attribute. If the parameter is set to + False or something not type of ParameterAttribute, + no bias is defined. If the parameter is set to + True, the bias is initialized to zero. + :type bias_attr: ParameterAttribute|None|Bool|Any :return: LayerOutput object. :rtype: LayerOutput """ @@ -2301,9 +2312,11 @@ def hsigmoid(input, :type num_classes: int|None :param name: layer name :type name: basestring - :param bias_attr: Bias attribute. None means default bias. - False means no bias. - :type bias_attr: ParameterAttribute|False + :param bias_attr: The Bias Attribute. If the parameter is set to + False or something not type of ParameterAttribute, + no bias is defined. If the parameter is set to + True, the bias is initialized to zero. + :type bias_attr: ParameterAttribute|None|Bool|Any :param param_attr: Parameter Attribute. None means default parameter. :type param_attr: ParameterAttribute|None :param layer_attr: Extra Layer Attribute. @@ -2442,9 +2455,11 @@ def img_conv_layer(input, :type dilation: int|tuple|list :param dilation_y: The y dimension of the dilation. :type dilation_y: int - :param bias_attr: Convolution bias attribute. None means default bias. - False means no bias. - :type bias_attr: ParameterAttribute|False + :param bias_attr: The Bias Attribute. If the parameter is set to + False or something not type of ParameterAttribute, + no bias is defined. If the parameter is set to + True, the bias is initialized to zero. + :type bias_attr: ParameterAttribute|None|Bool|Any :param num_channels: number of input channels. If None will be set automatically from previous output. :type num_channels: int @@ -3016,7 +3031,7 @@ def batch_norm_layer(input, :type num_channels: int :param bias_attr: :math:`\\beta`, better be zero when initialize. So the initial_std=0, initial_mean=1 is best practice. - :type bias_attr: ParameterAttribute + :type bias_attr: ParameterAttribute|None|Bool|Any :param param_attr: :math:`\\gamma`, better be one when initialize. So the initial_std=0, initial_mean=1 is best practice. :type param_attr: ParameterAttribute @@ -3186,9 +3201,11 @@ def addto_layer(input, act=None, name=None, bias_attr=None, layer_attr=None): :type input: LayerOutput|list|tuple :param act: Activation Type, default is tanh. :type act: BaseActivation - :param bias_attr: Bias attribute. If False, means no bias. None is default - bias. - :type bias_attr: ParameterAttribute|bool + :param bias_attr: The Bias Attribute. If the parameter is set to + False or something not type of ParameterAttribute, + no bias is defined. If the parameter is set to + True, the bias is initialized to zero. + :type bias_attr: ParameterAttribute|None|Bool|Any :param layer_attr: Extra Layer attribute. :type layer_attr: ExtraLayerAttribute :return: LayerOutput object. @@ -3340,10 +3357,11 @@ def seq_concat_layer(a, b, act=None, name=None, layer_attr=None, :type act: BaseActivation :param layer_attr: Extra Layer Attribute. :type layer_attr: ExtraLayerAttribute - :param bias_attr: The Bias Attribute. If no bias, then pass False or - something not type of ParameterAttribute. None will get a - default Bias. - :type bias_attr: ParameterAttribute or None or bool + :param bias_attr: The Bias Attribute. If the parameter is set to + False or something not type of ParameterAttribute, + no bias is defined. If the parameter is set to + True, the bias is initialized to zero. + :type bias_attr: ParameterAttribute|None|Bool|Any :return: LayerOutput object. :rtype: LayerOutput """ @@ -3524,8 +3542,11 @@ def lstm_step_layer(input, :param state_act: State Activation Type. Default is sigmoid, and should be sigmoid only. :type state_act: BaseActivation - :param bias_attr: Bias Attribute. - :type bias_attr: ParameterAttribute + :param bias_attr: The Bias Attribute. If the parameter is set to + False or something not type of ParameterAttribute, + no bias is defined. If the parameter is set to + True, the bias is initialized to zero. + :type bias_attr: ParameterAttribute|None|Bool|Any :param layer_attr: layer's extra attribute. :type layer_attr: ExtraLayerAttribute :return: LayerOutput object. @@ -3578,7 +3599,11 @@ def gru_step_layer(input, :param act: :param name: :param gate_act: - :param bias_attr: + :param bias_attr: The Bias Attribute. If the parameter is set to + False or something not type of ParameterAttribute, + no bias is defined. If the parameter is set to + True, the bias is initialized to zero. + :type bias_attr: ParameterAttribute|None|Bool|Any :param param_attr: the parameter_attribute for transforming the output_mem from previous step. :param layer_attr: @@ -3635,7 +3660,11 @@ def gru_step_naive_layer(input, :param name: :param act: :param gate_act: - :param bias_attr: + :param bias_attr: The Bias Attribute. If the parameter is set to + False or something not type of ParameterAttribute, + no bias is defined. If the parameter is set to + True, the bias is initialized to zero. + :type bias_attr: ParameterAttribute|None|Bool|Any :param param_attr: :param layer_attr: :return: @@ -3757,8 +3786,11 @@ def recurrent_layer(input, :type input: LayerOutput :param act: activation. :type act: BaseActivation - :param bias_attr: bias attribute. - :type bias_attr: ParameterAttribute + :param bias_attr: The Bias Attribute. If the parameter is set to + False or something not type of ParameterAttribute, + no bias is defined. If the parameter is set to + True, the bias is initialized to zero. + :type bias_attr: ParameterAttribute|None|Bool|Any :param param_attr: parameter attribute. :type param_attr: ParameterAttribute :param name: name of the layer @@ -4747,10 +4779,11 @@ def tensor_layer(a, :type act: BaseActivation :param param_attr: The Parameter Attribute. :type param_attr: ParameterAttribute - :param bias_attr: The Bias Attribute. If no bias, then pass False or - something not type of ParameterAttribute. None will get a - default Bias. - :type bias_attr: ParameterAttribute|None|Any + :param bias_attr: The Bias Attribute. If the parameter is set to + False or something not type of ParameterAttribute, + no bias is defined. If the parameter is set to + True, the bias is initialized to zero. + :type bias_attr: ParameterAttribute|None|Bool|Any :param layer_attr: Extra Layer config. :type layer_attr: ExtraLayerAttribute|None :return: LayerOutput object. @@ -4811,10 +4844,11 @@ def selective_fc_layer(input, :type act: BaseActivation :param param_attr: The Parameter Attribute. :type param_attr: ParameterAttribute - :param bias_attr: The Bias Attribute. If no bias, then pass False or - something not type of ParameterAttribute. None will get a - default Bias. - :type bias_attr: ParameterAttribute|None|Any + :param bias_attr: The Bias Attribute. If the parameter is set to + False or something not type of ParameterAttribute, + no bias is defined. If the parameter is set to + True, the bias is initialized to zero. + :type bias_attr: ParameterAttribute|None|Bool|Any :param layer_attr: Extra Layer config. :type layer_attr: ExtraLayerAttribute|None :return: LayerOutput object. @@ -5478,10 +5512,11 @@ def nce_layer(input, A uniform distribution will be used if not provided. If not None, its length must be equal to num_classes. :type neg_distribution: list|tuple|collections.Sequence|None - :param bias_attr: The Bias Attribute. If no bias, then pass False or - something not type of ParameterAttribute. None will get a - default Bias. - :type bias_attr: ParameterAttribute|None|False + :param bias_attr: The Bias Attribute. If the parameter is set to + False or something not type of ParameterAttribute, + no bias is defined. If the parameter is set to + True, the bias is initialized to zero. + :type bias_attr: ParameterAttribute|None|Bool|Any :param layer_attr: Extra Layer Attribute. :type layer_attr: ExtraLayerAttribute :return: layer name. @@ -6763,7 +6798,7 @@ def img_conv3d_layer(input, :type padding: int|tuple|list :param bias_attr: Convolution bias attribute. None means default bias. False means no bias. - :type bias_attr: ParameterAttribute|False + :type bias_attr: ParameterAttribute|None|Bool|Any :param num_channels: number of input channels. If None will be set automatically from previous output. :type num_channels: int @@ -6881,10 +6916,11 @@ def scale_shift_layer(input, name=None, param_attr=None, bias_attr=None): :type input: LayerOutput. :param param_attr: The parameter attribute of scaling. :type param_attr: ParameterAttribute - :param bias_attr: The Bias Attribute. If no bias, then pass False or - something not type of ParameterAttribute. None will get a - default Bias. - :type bias_attr: ParameterAttribute + :param bias_attr: The Bias Attribute. If the parameter is set to + False or something not type of ParameterAttribute, + no bias is defined. If the parameter is set to + True, the bias is initialized to zero. + :type bias_attr: ParameterAttribute|None|Bool|Any :return: LayerOutput object. :rtype: LayerOutput """ From 716e020e8084897bfff10c61e2583cf4bf8b1b42 Mon Sep 17 00:00:00 2001 From: ranqiu Date: Tue, 19 Sep 2017 17:36:33 +0800 Subject: [PATCH 14/35] Update faq about PaddlePaddle params --- doc/faq/index_cn.rst | 50 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/doc/faq/index_cn.rst b/doc/faq/index_cn.rst index 138efb566e..119e037aa2 100644 --- a/doc/faq/index_cn.rst +++ b/doc/faq/index_cn.rst @@ -321,3 +321,53 @@ pip uninstall py_paddle paddle 然后安装paddle的python环境, 在build目录下执行 pip install python/dist/paddle*.whl && pip install ../paddle/dist/py_paddle*.whl + +16. 如何加载预训练embedding参数 +------------------------------ + +设置embedding的参数属性 :code:`is_static=True`，使embedding参数在训练过程中保持不变，在创建parameters后，使用 :code:`parameters.set()` 加载预训练参数。 + +.. code-block:: python + + def load_parameter(file_name, h, w): + with open(file_name, 'rb') as f: + f.read(16) # skip header. + return np.fromfile(f, dtype=np.float32).reshape(h, w) + + + emb_para = paddle.attr.Param(name='emb', initial_std=0., is_static=True) + paddle.layer.embedding(size=word_dim, input=x, param_attr=emb_para) + + + parameters = paddle.parameters.create(my_cost) + parameters.set('emb', load_parameter(emb_param_file, 30000, 256)) + + +17. PaddlePaddle存储的参数格式是什么，如何和明文进行相互转化 +--------------------------------------------------------- + +PaddlePaddle保存的二进制参数文件内容由16位头信息和网络参数两部分组成。头信息中，第一位固定为0，第二位为4，在使用double精度时，第二位为8，第三位记录共有多少个数值。 + +将PaddlePaddle保存的二进制参数还原回明文时，先跳过PaddlePaddle模型参数文件的头信息，再提取网络参数，示例如下： + +.. code-block:: python + + def read_parameter(fname, width): + s = open(fname).read() + # skip header + vec = np.fromstring(s[16:], dtype=np.float32) + # width is the size of the corresponding layer + np.savetxt(fname + ".csv", vec.reshape(width, -1), + fmt="%.6f", delimiter=",") + + +将明文参数转化为PaddlePaddle可加载的模型参数时，先根据参数规模写入头信息，再写入具体网络参数。以下为将随机生成的矩阵转化为PaddlePaddle可加载的模型参数示例: + +.. code-block:: python + + def gen_rand_param(param_file, width, height, need_trans): + np.random.seed() + header = struct.pack("iil", 0, 4, height * width) + param = np.float32(np.random.rand(height, width)) + with open(param_file, "w") as fparam: + fparam.write(header + param.tostring()) From 70141a73c4cd5f3855a72c6488a6cac288bda36f Mon Sep 17 00:00:00 2001 From: Luo Tao Date: Tue, 19 Sep 2017 18:45:47 +0800 Subject: [PATCH 15/35] remove unused XXXLayer.h in Layer.cpp --- paddle/gserver/layers/Layer.cpp | 29 +---------------------------- 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/paddle/gserver/layers/Layer.cpp b/paddle/gserver/layers/Layer.cpp index 2bc20eee6c..f11875b176 100644 --- a/paddle/gserver/layers/Layer.cpp +++ b/paddle/gserver/layers/Layer.cpp @@ -14,26 +14,11 @@ limitations under the License. */ #include "paddle/utils/Util.h" +#include "Layer.h" #include "paddle/math/SparseMatrix.h" #include "paddle/utils/Error.h" #include "paddle/utils/Logging.h" -#include "AddtoLayer.h" -#include "CRFLayer.h" -#include "CosSimLayer.h" -#include "CostLayer.h" -#include "DataLayer.h" -#include "ExpandConvLayer.h" -#include "FullyConnectedLayer.h" -#include "HierarchicalSigmoidLayer.h" -#include "MaxLayer.h" -#include "MixedLayer.h" -#include "NormLayer.h" -#include "PoolLayer.h" -#include "TensorLayer.h" -#include "TransLayer.h" -#include "ValidationLayer.h" - DEFINE_bool(log_error_clipping, false, "enable log error clipping or not"); namespace paddle { @@ -108,18 +93,6 @@ ClassRegistrar Layer::registrar_; LayerPtr Layer::create(const LayerConfig& config) { std::string type = config.type(); - - if (type == "multi-class-cross-entropy") - return LayerPtr(new MultiClassCrossEntropy(config)); - else if (type == "rank-cost") - return LayerPtr(new RankingCost(config)); - else if (type == "auc-validation") - return LayerPtr(new AucValidation(config)); - else if (type == "pnpair-validation") - return LayerPtr(new PnpairValidation(config)); - // NOTE: stop adding "if" statements here. - // Instead, use REGISTER_LAYER to add more layer types - return LayerPtr(registrar_.createByType(config.type(), config)); } From fe2c5936d985a41528554445212e638a90c3120a Mon Sep 17 00:00:00 2001 From: ranqiu Date: Tue, 19 Sep 2017 19:15:22 +0800 Subject: [PATCH 16/35] Update annotation of layers.py --- python/paddle/trainer_config_helpers/layers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index 53aec16906..94769b40f2 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -6726,7 +6726,7 @@ def kmax_seq_score_layer(input, name=None, beam_size=1): :param input: The input layer. It stores scores over a sequence or a nested sequence and its size must be 1. :type input: LayerOutput. - :param beam_size: squence indices with top beam_size scores are returned. + :param beam_size: sequence indices with top beam_size scores are returned. :type beam_size: double :return: LayerOutput object. :rtype: LayerOutput From 62377fd1f3c09bcac8b49f970891a319c0055eeb Mon Sep 17 00:00:00 2001 From: ranqiu Date: Tue, 19 Sep 2017 20:17:05 +0800 Subject: [PATCH 17/35] Update annotations about layer name of layers.py --- .../paddle/trainer_config_helpers/layers.py | 128 +++++++++--------- 1 file changed, 64 insertions(+), 64 deletions(-) diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index 94769b40f2..c97e6c0a36 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -921,7 +921,7 @@ def data_layer(name, size, depth=None, height=None, width=None, data = data_layer(name="input", size=1000) - :param name: Name of this data layer. + :param name: The name of this layer. It is optional. :type name: basestring :param size: Size of this data layer. :type size: int @@ -961,7 +961,7 @@ def embedding_layer(input, size, name=None, param_attr=None, layer_attr=None): """ Define a embedding Layer. - :param name: Name of this embedding layer. + :param name: The name of this layer. It is optional. :type name: basestring :param input: The input layer for this embedding. NOTE: must be Index Data. :type input: LayerOutput @@ -1016,7 +1016,7 @@ def fc_layer(input, with mixed_layer(size=1024) as fc: fc += full_matrix_projection(input=layer) - :param name: The Layer Name. + :param name: The name of this layer. It is optional. :type name: basestring :param input: The input layer. Could be a list/tuple of input layer. :type input: LayerOutput|list|tuple @@ -1067,7 +1067,7 @@ def printer_layer(input, format=None, name=None): """ Print the output value of input layers. This layer is useful for debugging. - :param name: The Layer Name. + :param name: The name of this layer. It is optional. :type name: basestring :param input: The input layer. Could be a list/tuple of input layer. :type input: LayerOutput|list|tuple @@ -1105,7 +1105,7 @@ def priorbox_layer(input, """ Compute the priorbox and set the variance. This layer is necessary for ssd. - :param name: The Layer Name. + :param name: The name of this layer. It is optional. :type name: basestring :param input: The input layer. :type input: LayerOutput @@ -1154,7 +1154,7 @@ def multibox_loss_layer(input_loc, """ Compute the location loss and the confidence loss for ssd. - :param name: The Layer Name. + :param name: The name of this layer. It is optional. :type name: basestring :param input_loc: The input predict locations. :type input_loc: LayerOutput | List of LayerOutput @@ -1229,7 +1229,7 @@ def detection_output_layer(input_loc, box location. The output's shape of this layer could be zero if there is no valid bounding box. - :param name: The Layer Name. + :param name: The name of this layer. It is optional. :type name: basestring :param input_loc: The input predict locations. :type input_loc: LayerOutput | List of LayerOutput. @@ -1301,7 +1301,7 @@ def cross_channel_norm_layer(input, name=None, param_attr=None): a conv layer's output and scale the output by a group of trainable factors which dimensions equal to the channel's number. - :param name: The Layer Name. + :param name: The name of this layer. It is optional. :type name: basestring :param input: The input layer. :type input: LayerOutput @@ -1366,7 +1366,7 @@ def pooling_layer(input, :param agg_level: AggregateLevel.TO_NO_SEQUENCE or AggregateLevel.TO_SEQUENCE :type agg_level: AggregateLevel - :param name: layer name. + :param name: The name of this layer. It is optional. :type name: basestring :param input: input layer name. :type input: LayerOutput @@ -1665,7 +1665,7 @@ def last_seq(input, seq = last_seq(input=layer) :param agg_level: Aggregated level - :param name: Layer name. + :param name: The name of this layer. It is optional. :type name: basestring :param input: Input layer name. :type input: LayerOutput @@ -1721,7 +1721,7 @@ def first_seq(input, seq = first_seq(input=layer) :param agg_level: aggregation level - :param name: Layer name. + :param name: The name of this layer. It is optional. :type name: basestring :param input: Input layer name. :type input: LayerOutput @@ -1800,7 +1800,7 @@ def expand_layer(input, :type input: LayerOutput :param expand_as: Expand as this layer's sequence info. :type expand_as: LayerOutput - :param name: Layer name. + :param name: The name of this layer. It is optional. :type name: basestring :param bias_attr: The Bias Attribute. If the parameter is set to False or something not type of ParameterAttribute, @@ -1859,7 +1859,7 @@ def repeat_layer(input, :type input: LayerOutput :param num_repeats: Repeat the input so many times :type num_repeats: int - :param name: Layer name. + :param name: The name of this layer. It is optional. :param as_row_vector: True for treating input as row vector and repeating in the column direction. This is equivalent to apply concat_layer() with num_repeats same input. @@ -1918,7 +1918,7 @@ def seq_reshape_layer(input, :type input: LayerOutput :param reshape_size: the size of reshaped sequence. :type reshape_size: int - :param name: Layer name. + :param name: The name of this layer. It is optional. :type name: basestring :param act: Activation type. :type act: BaseActivation @@ -1971,7 +1971,7 @@ def interpolation_layer(input, weight, name=None, layer_attr=None): :type input: list|tuple :param weight: Weight layer. :type weight: LayerOutput - :param name: Layer name. + :param name: The name of this layer. It is optional. :type name: basestring :param layer_attr: extra layer attributes. :type layer_attr: ExtraLayerAttribute. @@ -2076,7 +2076,7 @@ def power_layer(input, weight, name=None, layer_attr=None): :type input: LayerOutput :param weight: Weight layer. :type weight: LayerOutput - :param name: Layer name. + :param name: The name of this layer. It is optional. :type name: basestring :param layer_attr: extra layer attributes. :type layer_attr: ExtraLayerAttribute. @@ -2120,7 +2120,7 @@ def scaling_layer(input, weight, name=None, layer_attr=None): :type input: LayerOutput :param weight: Weight layer. :type weight: LayerOutput - :param name: Layer name. + :param name: The name of this layer. It is optional. :type name: basestring :param layer_attr: extra layer attributes. :type layer_attr: ExtraLayerAttribute. @@ -2158,7 +2158,7 @@ def trans_layer(input, name=None, layer_attr=None): :param input: Input layer. :type input: LayerOutput - :param name: Layer name. + :param name: The name of this layer. It is optional. :type name: basestring :param layer_attr: extra layer attributes. :type layer_attr: ExtraLayerAttribute. @@ -2198,7 +2198,7 @@ def rotate_layer(input, height, width, name=None, layer_attr=None): :type input: LayerOutput :param height: The height of the sample matrix :type height: int - :param name: Layer name. + :param name: The name of this layer. It is optional. :type name: basestring :param layer_attr: extra layer attributes. :type layer_attr: ExtraLayerAttribute. @@ -2243,7 +2243,7 @@ def cos_sim(a, b, scale=1, size=1, name=None, layer_attr=None): cos = cos_sim(a=layer1, b=layer2, size=3) - :param name: layer name + :param name: The name of this layer. It is optional. :type name: basestring :param a: input layer a :type a: LayerOutput @@ -2310,7 +2310,7 @@ def hsigmoid(input, :type label: LayerOutput :param num_classes: number of classes. :type num_classes: int|None - :param name: layer name + :param name: The name of this layer. It is optional. :type name: basestring :param bias_attr: The Bias Attribute. If the parameter is set to False or something not type of ParameterAttribute, @@ -2424,7 +2424,7 @@ def img_conv_layer(input, bias_attr=False, act=ReluActivation()) - :param name: Layer name. + :param name: The name of this layer. It is optional. :type name: basestring :param input: Layer Input. :type input: LayerOutput @@ -2850,7 +2850,7 @@ def spp_layer(input, num_channels=16, pool_type=MaxPooling()) - :param name: layer name. + :param name: The name of this layer. It is optional. :type name: basestring :param input: layer's input. :type input: LayerOutput @@ -2944,7 +2944,7 @@ def img_cmrnorm_layer(input, norm = img_cmrnorm_layer(input=net, size=5) - :param name: layer name. + :param name: The name of this layer. It is optional. :type name: None|basestring :param input: layer's input. :type input: LayerOutput @@ -3007,7 +3007,7 @@ def batch_norm_layer(input, norm = batch_norm_layer(input=net, act=ReluActivation()) - :param name: layer name. + :param name: The name of this layer. It is optional. :type name: basestring :param input: batch normalization input. Better be linear activation. Because there is an activation inside batch_normalization. @@ -3106,7 +3106,7 @@ def sum_to_one_norm_layer(input, name=None, layer_attr=None): :param input: Input layer. :type input: LayerOutput - :param name: Layer name. + :param name: The name of this layer. It is optional. :type name: basestring :param layer_attr: extra layer attributes. :type layer_attr: ExtraLayerAttribute. @@ -3142,7 +3142,7 @@ def row_l2_norm_layer(input, name=None, layer_attr=None): :param input: Input layer. :type input: LayerOutput - :param name: Layer name. + :param name: The name of this layer. It is optional. :type name: basestring :param layer_attr: extra layer attributes. :type layer_attr: ExtraLayerAttribute. @@ -3194,7 +3194,7 @@ def addto_layer(input, act=None, name=None, bias_attr=None, layer_attr=None): dropout here. Please refer to dropout_layer for details. - :param name: Layer name. + :param name: The name of this layer. It is optional. :type name: basestring :param input: Input layers. It could be a LayerOutput or list/tuple of LayerOutput. @@ -3254,7 +3254,7 @@ def concat_layer(input, act=None, name=None, layer_attr=None, bias_attr=None): concat = concat_layer(input=[layer1, layer2]) - :param name: Layer name. + :param name: The name of this layer. It is optional. :type name: basestring :param input: input layers or projections :type input: list|tuple|collections.Sequence @@ -3347,7 +3347,7 @@ def seq_concat_layer(a, b, act=None, name=None, layer_attr=None, concat = seq_concat_layer(a=layer1, b=layer2) - :param name: Layer name. + :param name: The name of this layer. It is optional. :type name: basestring :param a: input sequence layer :type a: LayerOutput @@ -3524,7 +3524,7 @@ def lstm_step_layer(input, output is :math:`o_t`, whose name is 'state' and can use :code:`get_output_layer` to extract this output. - :param name: Layer's name. + :param name: The name of this layer. It is optional. :type name: basestring :param size: Layer's size. NOTE: lstm layer's size, should be equal to :code:`input.size/4`, and should be equal to @@ -3597,7 +3597,7 @@ def gru_step_layer(input, :param output_mem: :param size: :param act: - :param name: + :param name: The name of this layer. It is optional. :param gate_act: :param bias_attr: The Bias Attribute. If the parameter is set to False or something not type of ParameterAttribute, @@ -3657,7 +3657,7 @@ def gru_step_naive_layer(input, :param input: :param output_mem: :param size: - :param name: + :param name: The name of this layer. It is optional. :param act: :param gate_act: :param bias_attr: The Bias Attribute. If the parameter is set to @@ -3720,7 +3720,7 @@ def get_output_layer(input, arg_name, name=None, layer_attr=None): output besides the default one, please use get_output_layer first to get the output from input. - :param name: Layer's name. + :param name: The name of this layer. It is optional. :type name: basestring :param input: get output layer's input. And this layer should contains multiple outputs. @@ -3793,7 +3793,7 @@ def recurrent_layer(input, :type bias_attr: ParameterAttribute|None|Bool|Any :param param_attr: parameter attribute. :type param_attr: ParameterAttribute - :param name: name of the layer + :param name: The name of this layer. It is optional. :type name: basestring :param layer_attr: Layer Attribute. :type layer_attr: ExtraLayerAttribute @@ -4032,7 +4032,7 @@ def maxid_layer(input, name=None, layer_attr=None): :param input: Input layer name. :type input: LayerOutput - :param name: Layer name. + :param name: The name of this layer. It is optional. :type name: basestring :param layer_attr: extra layer attributes. :type layer_attr: ExtraLayerAttribute. @@ -4065,7 +4065,7 @@ def out_prod_layer(input1, input2, name=None, layer_attr=None): out_prod = out_prod_layer(input1=vec1, input2=vec2) - :param name: Layer name. + :param name: The name of this layer. It is optional. :type name: basestring :param input1: The first input layer name. :type input: LayerOutput @@ -4106,7 +4106,7 @@ def eos_layer(input, eos_id, name=None, layer_attr=None): eos = eos_layer(input=layer, eos_id=id) - :param name: Layer name. + :param name: The name of this layer. It is optional. :type name: basestring :param input: Input layer name. :type input: LayerOutput @@ -4297,7 +4297,7 @@ def square_error_cost(input, cost = \\sum_{i=1}^N(t_i-y_i)^2 - :param name: layer name. + :param name: The name of this layer. It is optional. :type name: basestring :param input: Network prediction. :type input: LayerOutput @@ -4339,7 +4339,7 @@ def classification_cost(input, """ classification cost Layer. - :param name: layer name. + :param name: The name of this layer. It is optional. :type name: basestring :param input: input layer name. network output. :type input: LayerOutput @@ -4643,7 +4643,7 @@ def pad_layer(input, :type pad_w: list|None :param layer_attr: Extra Layer Attribute. :type layer_attr: ExtraLayerAttribute - :param name: layer name. + :param name: The name of this layer. It is optional. :type name: basestring :return: LayerOutput object. :rtype: LayerOutput @@ -4711,7 +4711,7 @@ def conv_shift_layer(a, b, name=None, layer_attr=None): conv_shift = conv_shift_layer(a=layer1, b=layer2) - :param name: layer name + :param name: The name of this layer. It is optional. :type name: basestring :param a: Input layer a. :type a: LayerOutput @@ -4767,7 +4767,7 @@ def tensor_layer(a, tensor = tensor_layer(a=layer1, b=layer2, size=1000) - :param name: layer name + :param name: The name of this layer. It is optional. :type name: basestring :param a: Input layer a. :type a: LayerOutput @@ -4830,7 +4830,7 @@ def selective_fc_layer(input, sel_fc = selective_fc_layer(input=input, size=128, act=TanhActivation()) - :param name: The Layer Name. + :param name: The name of this layer. It is optional. :type name: basestring :param input: The input layer. :type input: LayerOutput|list|tuple @@ -4904,7 +4904,7 @@ def sampling_id_layer(input, name=None, layer_attr=None): :param input: The input layer. :type input: LayerOutput - :param name: The Layer Name. + :param name: The name of this layer. It is optional. :type name: basestring :param layer_attr: Extra Layer config. :type layer_attr: ExtraLayerAttribute|None @@ -4942,7 +4942,7 @@ def slope_intercept_layer(input, :param input: The input layer. :type input: LayerOutput - :param name: The Layer Name. + :param name: The name of this layer. It is optional. :type name: basestring :param slope: the scale factor. :type slope: float. @@ -5006,7 +5006,7 @@ def linear_comb_layer(weights, vectors, size=None, name=None, layer_attr=None): :type vectors: LayerOutput :param size: the dimension of this layer. :type size: int - :param name: The Layer Name. + :param name: The name of this layer. It is optional. :type name: basestring :param layer_attr: Extra Layer config. :type layer_attr: ExtraLayerAttribute|None @@ -5089,7 +5089,7 @@ def block_expand_layer(input, :type padding_x: int :param padding_y: The padding size in vertical direction. :type padding_y: int - :param name: The name of this layer, which can not specify. + :param name: The name of this layer. It is optional. :type name: None|basestring. :param layer_attr: Extra Layer config. :type layer_attr: ExtraLayerAttribute|None @@ -5158,7 +5158,7 @@ def maxout_layer(input, groups, num_channels=None, name=None, layer_attr=None): :type num_channels: int|None :param groups: The group number of input layer. :type groups: int - :param name: The name of this layer, which can not specify. + :param name: The name of this layer. It is optional. :type name: None|basestring. :param layer_attr: Extra Layer attribute. :type layer_attr: ExtraLayerAttribute @@ -5222,7 +5222,7 @@ def ctc_layer(input, :type label: LayerOutput :param size: category numbers + 1. :type size: int - :param name: The name of this layer + :param name: The name of this layer. It is optional. :type name: basestring|None :param norm_by_times: Whether to normalization by times. False by default. :type norm_by_times: bool @@ -5299,7 +5299,7 @@ def warp_ctc_layer(input, :type label: LayerOutput :param size: category numbers + 1. :type size: int - :param name: The name of this layer, which can not specify. + :param name: The name of this layer. It is optional. :type name: basestring|None :param blank: the 'blank' label used in ctc :type blank: int @@ -5492,7 +5492,7 @@ def nce_layer(input, param_attr=[attr1, attr2], weight=layer3, num_classes=3, neg_distribution=[0.1,0.3,0.6]) - :param name: The name of this layer. + :param name: The name of this layer. It is optional. :type name: basestring :param input: The input layers. It could be a LayerOutput of list/tuple of LayerOutput. :type input: LayerOutput|list|tuple|collections.Sequence @@ -6182,7 +6182,7 @@ def multiplex_layer(input, name=None, layer_attr=None): :param input: Input layers. :type input: list of LayerOutput - :param name: Layer name. + :param name: The name of this layer. It is optional. :type name: basestring :param layer_attr: extra layer attributes. :type layer_attr: ExtraLayerAttribute. @@ -6220,7 +6220,7 @@ def dropout_layer(input, dropout_rate, name=None): dropout = dropout_layer(input=input_layer, dropout_rate=0.5) - :param name: The name of this layer. + :param name: The name of this layer. It is optional. :type name: basestring :param input: The input layer. :type input: LayerOutput @@ -6336,7 +6336,7 @@ def prelu_layer(input, prelu = prelu_layer(input=layers, partial_sum=1) - :param name: Name of this layer. + :param name: The name of this layer. It is optional. :type name: basestring :param input: The input layer. :type input: LayerOutput @@ -6409,7 +6409,7 @@ def gated_unit_layer(input, :type size: int :param act: activation type of the projected input. :type act: BaseActivation - :param name: name of this layer. + :param name: The name of this layer. It is optional. :type name: basestring :param gate_attr: Attributes to tune the gate output, for example, error clipping threshold, dropout and so on. See ExtraLayerAttribute for @@ -6485,7 +6485,7 @@ def switch_order_layer(input, :param input: The input layer. :type input: LayerOutput - :param name: Name of this layer. + :param name: The name of this layer. It is optional. :type name: basestring :param reshape_axis: Specify the axises of 'height'. Its value should be positive and less than 4. :type reshape_axis: int @@ -6538,7 +6538,7 @@ def crop_layer(input, offset, axis=2, shape=None, name=None, layer_attr=None): :type partial_sum: int :param shape: The shape to be cropped. Default is None. :type shape: Sequence | None - :param name: Name of this layer. + :param name: The name of this layer. It is optional. :type name: basestring :return: LayerOutput object. :rtype: LayerOutput @@ -6584,7 +6584,7 @@ def sub_nested_seq_layer(input, selected_indices, name=None): :type input: LayerOutput :param selected_indices: a set of sequence indices in the nested sequence. :type input: LayerOutput - :param name: name of this layer. + :param name: The name of this layer. It is optional. :type name: basestring :return: LayerOutput object. :rtype: LayerOutput @@ -6622,7 +6622,7 @@ def clip_layer(input, min, max, name=None): clip = clip_layer(input=input_layer, min=-10, max=10) - :param name: The Layer Name. + :param name: The name of this layer. It is optional. :type name: basestring :param input: The input layer. :type input: LayerOutput. @@ -6667,7 +6667,7 @@ def seq_slice_layer(input, starts, ends, name=None): seq_silce = seq_slice_layer(input=input_seq, starts=start_pos, ends=end_pos) - :param name: name of this layer. + :param name: The name of this layer. It is optional. :type name: basestring :param input: input for this layer, it should be a sequence. :type input: LayerOutput @@ -6721,7 +6721,7 @@ def kmax_seq_score_layer(input, name=None, beam_size=1): kmax_indices = kmax_seq_score_layer(input=input_layer, beam_size) - :param name: The Layer Name. + :param name: The name of this layer. It is optional. :type name: basestring :param input: The input layer. It stores scores over a sequence or a nested sequence and its size must be 1. @@ -6779,7 +6779,7 @@ def img_conv3d_layer(input, bias_attr=False, act=ReluActivation()) - :param name: Layer name. + :param name: The name of this layer. It is optional. :type name: basestring :param input: Layer Input. :type input: LayerOutput @@ -6910,7 +6910,7 @@ def scale_shift_layer(input, name=None, param_attr=None, bias_attr=None): scale_shift = scale_shift_layer(input=input_layer, bias_attr=False) - :param name: The Layer Name. + :param name: The name of this layer. It is optional. :type name: basestring :param input: The input layer. :type input: LayerOutput. From 3a4897ab155e4a71dcec25b2215fa3765a6af512 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Tue, 19 Sep 2017 11:27:47 -0700 Subject: [PATCH 18/35] Add TensorCopy method A method to copy a tensor with stride and dimension. It is useful for Crop, Concat, etc. --- paddle/operators/CMakeLists.txt | 1 + paddle/operators/detail/tensor_copy.h | 93 +++++++++++++++++++++++++++ paddle/operators/tensor_copy.h | 43 +++++++++++++ paddle/operators/tensor_copy_test.cc | 77 ++++++++++++++++++++++ 4 files changed, 214 insertions(+) create mode 100644 paddle/operators/detail/tensor_copy.h create mode 100644 paddle/operators/tensor_copy.h create mode 100644 paddle/operators/tensor_copy_test.cc diff --git a/paddle/operators/CMakeLists.txt b/paddle/operators/CMakeLists.txt index e3e934bccc..95f0acace9 100644 --- a/paddle/operators/CMakeLists.txt +++ b/paddle/operators/CMakeLists.txt @@ -96,3 +96,4 @@ set(GLOB_OP_LIB ${OP_LIBRARY} CACHE INTERNAL "Global OP library") cc_test(gather_test SRCS gather_test.cc DEPS tensor) cc_test(net_op_test SRCS net_op_test.cc DEPS net_op) cc_test(scatter_test SRCS scatter_test.cc DEPS tensor) +cc_test(tensor_copy_test SRCS tensor_copy_test.cc DEPS tensor paddle_memory) diff --git a/paddle/operators/detail/tensor_copy.h b/paddle/operators/detail/tensor_copy.h new file mode 100644 index 0000000000..44fe495648 --- /dev/null +++ b/paddle/operators/detail/tensor_copy.h @@ -0,0 +1,93 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once +#include "paddle/framework/ddim.h" +#include "paddle/memory/memcpy.h" +#include "paddle/platform/device_context.h" + +namespace paddle { +namespace operators { +namespace detail { + +template +struct TensorCopyFunctor; + +template +struct TensorCopyFunctor { + void operator()(const platform::DeviceContext& dev_ctx, const T* src, + framework::Dim<1> src_stride, framework::Dim<1> dst_dim, + framework::Dim<1> dst_stride, T* dst) const { + auto place = dev_ctx.GetPlace(); + if (platform::is_cpu_place(place)) { + auto& cpu_place = boost::get(place); + memory::Copy(cpu_place, dst, cpu_place, src, sizeof(T) * dst_dim.head); + } else { +#ifndef PADDLE_ONLY_CPU + auto& gpu_place = boost::get(place); + auto& cuda_ctx = + reinterpret_cast(dev_ctx); + memory::Copy(gpu_place, dst, gpu_place, src, sizeof(T) * dst_dim.head, + cuda_ctx.stream()); +#else + PADDLE_THROW("Paddle is not compiled with GPU"); +#endif + } + } +}; + +template +struct TensorCopyFunctor { + void operator()(const platform::DeviceContext& dev_ctx, const T* src, + framework::Dim src_stride, framework::Dim dst_dim, + framework::Dim dst_stride, T* dst) const { + for (int64_t i = 0; i < dst_dim.head; ++i) { + TensorCopyFunctor func; + func(dev_ctx, src, src_stride.tail, dst_dim.tail, dst_stride.tail, dst); + src += src_stride.head; + dst += dst_stride.head; + } + } +}; + +template +struct TensorCopyDimVisitor : public boost::static_visitor { + TensorCopyDimVisitor(const platform::DeviceContext& dev_ctx, const T* src, + const framework::DDim& src_stride, + const framework::DDim& dst_stride, T* dst) + : dev_ctx_(dev_ctx), + src_(src), + src_stride_(src_stride), + dst_stride_(dst_stride), + dst_(dst) {} + + template + void operator()(Dim dst_dim) const { + Dim src_stride = boost::get(src_stride_); + Dim dst_stride = boost::get(dst_stride_); + constexpr int dim = Dim::dimensions; + TensorCopyFunctor functor; + functor(dev_ctx_, src_, src_stride, dst_dim, dst_stride, dst_); + } + + const platform::DeviceContext& dev_ctx_; + const T* src_; + const framework::DDim& src_stride_; + const framework::DDim& dst_stride_; + T* dst_; +}; + +} // namespace detail +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/tensor_copy.h b/paddle/operators/tensor_copy.h new file mode 100644 index 0000000000..9210b4638b --- /dev/null +++ b/paddle/operators/tensor_copy.h @@ -0,0 +1,43 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once +#include "paddle/operators/detail/tensor_copy.h" + +namespace paddle { +namespace operators { + +// Copy a tensor from src to dst. +// The src and dst should be both on dev_ctx.GetPlace() +// +// the stride of an array (also referred to as increment, pitch or step size) is +// the number of locations in memory between beginnings of successive array +// elements +// +// For example, for tensor like [1, 3, 300, 300]. If there is no padding, the +// stride is [270000, 90000, 300, 1]. +// +// NOTE: When use GPU, the memcpy is async. To sync memcpy, please invoke +// `dev_ctx.Wait()`. +template +inline void TensorCopy(const platform::DeviceContext& dev_ctx, const T* src, + const framework::DDim& src_stride, + const framework::DDim& dst_dim, + const framework::DDim& dst_stride, T* dst) { + using namespace detail; + TensorCopyDimVisitor func(dev_ctx, src, src_stride, dst_stride, dst); + boost::apply_visitor(func, dst_dim); +} +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/tensor_copy_test.cc b/paddle/operators/tensor_copy_test.cc new file mode 100644 index 0000000000..df177096d3 --- /dev/null +++ b/paddle/operators/tensor_copy_test.cc @@ -0,0 +1,77 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/tensor_copy.h" +#include "gtest/gtest.h" +#include "paddle/memory/memory.h" + +namespace paddle { +namespace operators { +TEST(TensorCopy, CPU_COPY) { + int src[] = { + 0, 1, 2, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0, + }; + + framework::DDim src_stride({5, 1}); + + int dst[4]; + framework::DDim dst_dim({2, 2}); + framework::DDim dst_stride({2, 1}); + + platform::CPUDeviceContext ctx; + TensorCopy(ctx, src + 1, src_stride, dst_dim, dst_stride, dst); + + ASSERT_EQ(1, dst[0]); + ASSERT_EQ(2, dst[1]); + ASSERT_EQ(3, dst[2]); + ASSERT_EQ(4, dst[3]); +} + +#ifndef PADDLE_ONLY_CPU +TEST(TensorCopy, GPU_COPY) { + int src[] = { + 0, 1, 2, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0, + }; + + platform::GPUPlace gpu0(0); + platform::CPUPlace cpu; + + int* gpu_src = reinterpret_cast(memory::Alloc(gpu0, sizeof(src))); + memory::Copy(gpu0, gpu_src, cpu, src, sizeof(src)); + + framework::DDim src_stride({5, 1}); + + int dst[4]; + int* gpu_dst = reinterpret_cast(memory::Alloc(gpu0, sizeof(dst))); + + framework::DDim dst_dim({2, 2}); + framework::DDim dst_stride({2, 1}); + + platform::CUDADeviceContext ctx(gpu0); + TensorCopy(ctx, gpu_src + 1, src_stride, dst_dim, dst_stride, gpu_dst); + + memory::Copy(cpu, dst, gpu0, gpu_dst, sizeof(dst)); + + ASSERT_EQ(1, dst[0]); + ASSERT_EQ(2, dst[1]); + ASSERT_EQ(3, dst[2]); + ASSERT_EQ(4, dst[3]); + + memory::Free(gpu0, gpu_dst); + memory::Free(gpu0, gpu_src); +} + +#endif +} // namespace operators +} // namespace paddle \ No newline at end of file From 07915c95ecb36c632e69fd5cee0cae09f2430735 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Tue, 19 Sep 2017 14:30:34 -0700 Subject: [PATCH 19/35] Renamed to strided_memcpy and prettify unittests Add unittests for Crop and Concat --- paddle/operators/CMakeLists.txt | 2 +- .../{tensor_copy.h => strided_memcpy.h} | 18 +- .../{tensor_copy.h => strided_memcpy.h} | 20 ++- paddle/operators/strided_memcpy_test.cc | 160 ++++++++++++++++++ paddle/operators/tensor_copy_test.cc | 77 --------- 5 files changed, 181 insertions(+), 96 deletions(-) rename paddle/operators/detail/{tensor_copy.h => strided_memcpy.h} (86%) rename paddle/operators/{tensor_copy.h => strided_memcpy.h} (65%) create mode 100644 paddle/operators/strided_memcpy_test.cc delete mode 100644 paddle/operators/tensor_copy_test.cc diff --git a/paddle/operators/CMakeLists.txt b/paddle/operators/CMakeLists.txt index 95f0acace9..90c7171419 100644 --- a/paddle/operators/CMakeLists.txt +++ b/paddle/operators/CMakeLists.txt @@ -96,4 +96,4 @@ set(GLOB_OP_LIB ${OP_LIBRARY} CACHE INTERNAL "Global OP library") cc_test(gather_test SRCS gather_test.cc DEPS tensor) cc_test(net_op_test SRCS net_op_test.cc DEPS net_op) cc_test(scatter_test SRCS scatter_test.cc DEPS tensor) -cc_test(tensor_copy_test SRCS tensor_copy_test.cc DEPS tensor paddle_memory) +cc_test(strided_memcpy_test SRCS strided_memcpy_test.cc DEPS tensor paddle_memory) diff --git a/paddle/operators/detail/tensor_copy.h b/paddle/operators/detail/strided_memcpy.h similarity index 86% rename from paddle/operators/detail/tensor_copy.h rename to paddle/operators/detail/strided_memcpy.h index 44fe495648..b165224b37 100644 --- a/paddle/operators/detail/tensor_copy.h +++ b/paddle/operators/detail/strided_memcpy.h @@ -22,10 +22,10 @@ namespace operators { namespace detail { template -struct TensorCopyFunctor; +struct StridedMemcpyFunctor; template -struct TensorCopyFunctor { +struct StridedMemcpyFunctor { void operator()(const platform::DeviceContext& dev_ctx, const T* src, framework::Dim<1> src_stride, framework::Dim<1> dst_dim, framework::Dim<1> dst_stride, T* dst) const { @@ -48,12 +48,12 @@ struct TensorCopyFunctor { }; template -struct TensorCopyFunctor { +struct StridedMemcpyFunctor { void operator()(const platform::DeviceContext& dev_ctx, const T* src, framework::Dim src_stride, framework::Dim dst_dim, framework::Dim dst_stride, T* dst) const { for (int64_t i = 0; i < dst_dim.head; ++i) { - TensorCopyFunctor func; + StridedMemcpyFunctor func; func(dev_ctx, src, src_stride.tail, dst_dim.tail, dst_stride.tail, dst); src += src_stride.head; dst += dst_stride.head; @@ -62,10 +62,10 @@ struct TensorCopyFunctor { }; template -struct TensorCopyDimVisitor : public boost::static_visitor { - TensorCopyDimVisitor(const platform::DeviceContext& dev_ctx, const T* src, - const framework::DDim& src_stride, - const framework::DDim& dst_stride, T* dst) +struct StridedCopyDimVisitor : public boost::static_visitor { + StridedCopyDimVisitor(const platform::DeviceContext& dev_ctx, const T* src, + const framework::DDim& src_stride, + const framework::DDim& dst_stride, T* dst) : dev_ctx_(dev_ctx), src_(src), src_stride_(src_stride), @@ -77,7 +77,7 @@ struct TensorCopyDimVisitor : public boost::static_visitor { Dim src_stride = boost::get(src_stride_); Dim dst_stride = boost::get(dst_stride_); constexpr int dim = Dim::dimensions; - TensorCopyFunctor functor; + StridedMemcpyFunctor functor; functor(dev_ctx_, src_, src_stride, dst_dim, dst_stride, dst_); } diff --git a/paddle/operators/tensor_copy.h b/paddle/operators/strided_memcpy.h similarity index 65% rename from paddle/operators/tensor_copy.h rename to paddle/operators/strided_memcpy.h index 9210b4638b..c9dd805184 100644 --- a/paddle/operators/tensor_copy.h +++ b/paddle/operators/strided_memcpy.h @@ -13,15 +13,17 @@ limitations under the License. */ #pragma once -#include "paddle/operators/detail/tensor_copy.h" +#include "paddle/operators/detail/strided_memcpy.h" namespace paddle { namespace operators { -// Copy a tensor from src to dst. -// The src and dst should be both on dev_ctx.GetPlace() +// Strided memory copy from src to dst. // -// the stride of an array (also referred to as increment, pitch or step size) is +// The src and dst should be both on dev_ctx.GetPlace(), otherwise, there will +// be a segment fault. +// +// The stride of an array (also referred to as increment, pitch or step size) is // the number of locations in memory between beginnings of successive array // elements // @@ -31,12 +33,12 @@ namespace operators { // NOTE: When use GPU, the memcpy is async. To sync memcpy, please invoke // `dev_ctx.Wait()`. template -inline void TensorCopy(const platform::DeviceContext& dev_ctx, const T* src, - const framework::DDim& src_stride, - const framework::DDim& dst_dim, - const framework::DDim& dst_stride, T* dst) { +inline void StridedMemcpy(const platform::DeviceContext& dev_ctx, const T* src, + const framework::DDim& src_stride, + const framework::DDim& dst_dim, + const framework::DDim& dst_stride, T* dst) { using namespace detail; - TensorCopyDimVisitor func(dev_ctx, src, src_stride, dst_stride, dst); + StridedCopyDimVisitor func(dev_ctx, src, src_stride, dst_stride, dst); boost::apply_visitor(func, dst_dim); } } // namespace operators diff --git a/paddle/operators/strided_memcpy_test.cc b/paddle/operators/strided_memcpy_test.cc new file mode 100644 index 0000000000..05882a8873 --- /dev/null +++ b/paddle/operators/strided_memcpy_test.cc @@ -0,0 +1,160 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/strided_memcpy.h" +#include "gtest/gtest.h" +#include "paddle/memory/memory.h" + +namespace paddle { +namespace operators { + +TEST(StridedMemcpy, CPUCrop) { + // clang-format off + int src[] = { + 0, 1, 2, 0, 0, + 0, 3, 4, 0, 0, + 0, 0, 0, 0, 0, + }; + // clang-format on + + framework::DDim src_stride({5, 1}); + + int dst[4]; + framework::DDim dst_dim({2, 2}); + framework::DDim dst_stride({2, 1}); + + platform::CPUDeviceContext ctx; + StridedMemcpy(ctx, src + 1, src_stride, dst_dim, dst_stride, dst); + + ASSERT_EQ(1, dst[0]); + ASSERT_EQ(2, dst[1]); + ASSERT_EQ(3, dst[2]); + ASSERT_EQ(4, dst[3]); +} + +TEST(StridedMemcpy, CPUConcat) { + // clang-format off + int src[] = { + 1, 2, + 3, 4 + }; + // clang-format on + + int dst[8]; + + framework::DDim src_stride({2, 1}); + framework::DDim dst_dim({2, 2}); + framework::DDim dst_stride({4, 1}); + platform::CPUDeviceContext ctx; + + StridedMemcpy(ctx, src, src_stride, dst_dim, dst_stride, dst); + StridedMemcpy(ctx, src, src_stride, dst_dim, dst_stride, dst + 2); + + // clang-format off + int expect_dst[] = { + 1, 2, 1, 2, + 3, 4, 3, 4 + }; + // clang-format on + for (size_t i = 0; i < sizeof(expect_dst) / sizeof(int); ++i) { + ASSERT_EQ(expect_dst[i], dst[i]); + } +} + +#ifndef PADDLE_ONLY_CPU +TEST(StridedMemcpy, GPUCrop) { + // clang-format off + int src[] = { + 0, 1, 2, 0, 0, + 0, 3, 4, 0, 0, + 0, 0, 0, 0, 0, + }; + // clang-format on + + platform::GPUPlace gpu0(0); + platform::CPUPlace cpu; + + int* gpu_src = reinterpret_cast(memory::Alloc(gpu0, sizeof(src))); + memory::Copy(gpu0, gpu_src, cpu, src, sizeof(src)); + + framework::DDim src_stride({5, 1}); + + int dst[4]; + int* gpu_dst = reinterpret_cast(memory::Alloc(gpu0, sizeof(dst))); + + framework::DDim dst_dim({2, 2}); + framework::DDim dst_stride({2, 1}); + + platform::CUDADeviceContext ctx(gpu0); + StridedMemcpy(ctx, gpu_src + 1, src_stride, dst_dim, dst_stride, + gpu_dst); + + memory::Copy(cpu, dst, gpu0, gpu_dst, sizeof(dst), ctx.stream()); + ctx.Wait(); + + ASSERT_EQ(1, dst[0]); + ASSERT_EQ(2, dst[1]); + ASSERT_EQ(3, dst[2]); + ASSERT_EQ(4, dst[3]); + + memory::Free(gpu0, gpu_dst); + memory::Free(gpu0, gpu_src); +} + +TEST(StridedMemcpy, GPUConcat) { + // clang-format off + int src[] = { + 1, 2, + 3, 4 + }; + // clang-format on + + platform::GPUPlace gpu0(0); + platform::CPUPlace cpu; + + int* gpu_src = reinterpret_cast(memory::Alloc(gpu0, sizeof(src))); + memory::Copy(gpu0, gpu_src, cpu, src, sizeof(src)); + + int dst[8]; + int* gpu_dst = reinterpret_cast(memory::Alloc(gpu0, sizeof(dst))); + + framework::DDim src_stride({2, 1}); + framework::DDim dst_dim({2, 2}); + framework::DDim dst_stride({4, 1}); + platform::CUDADeviceContext ctx(gpu0); + + StridedMemcpy(ctx, gpu_src, src_stride, dst_dim, dst_stride, gpu_dst); + StridedMemcpy(ctx, gpu_src, src_stride, dst_dim, dst_stride, + gpu_dst + 2); + + memory::Copy(cpu, dst, gpu0, gpu_dst, sizeof(dst), ctx.stream()); + ctx.Wait(); + + // clang-format off + int expect_dst[] = { + 1, 2, 1, 2, + 3, 4, 3, 4 + }; + // clang-format on + for (size_t i = 0; i < sizeof(expect_dst) / sizeof(int); ++i) { + ASSERT_EQ(expect_dst[i], dst[i]); + } + + memory::Free(gpu0, gpu_dst); + memory::Free(gpu0, gpu_src); +} + +#endif +} // namespace operators +} // namespace paddle \ No newline at end of file diff --git a/paddle/operators/tensor_copy_test.cc b/paddle/operators/tensor_copy_test.cc deleted file mode 100644 index df177096d3..0000000000 --- a/paddle/operators/tensor_copy_test.cc +++ /dev/null @@ -1,77 +0,0 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#include "paddle/operators/tensor_copy.h" -#include "gtest/gtest.h" -#include "paddle/memory/memory.h" - -namespace paddle { -namespace operators { -TEST(TensorCopy, CPU_COPY) { - int src[] = { - 0, 1, 2, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0, - }; - - framework::DDim src_stride({5, 1}); - - int dst[4]; - framework::DDim dst_dim({2, 2}); - framework::DDim dst_stride({2, 1}); - - platform::CPUDeviceContext ctx; - TensorCopy(ctx, src + 1, src_stride, dst_dim, dst_stride, dst); - - ASSERT_EQ(1, dst[0]); - ASSERT_EQ(2, dst[1]); - ASSERT_EQ(3, dst[2]); - ASSERT_EQ(4, dst[3]); -} - -#ifndef PADDLE_ONLY_CPU -TEST(TensorCopy, GPU_COPY) { - int src[] = { - 0, 1, 2, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0, - }; - - platform::GPUPlace gpu0(0); - platform::CPUPlace cpu; - - int* gpu_src = reinterpret_cast(memory::Alloc(gpu0, sizeof(src))); - memory::Copy(gpu0, gpu_src, cpu, src, sizeof(src)); - - framework::DDim src_stride({5, 1}); - - int dst[4]; - int* gpu_dst = reinterpret_cast(memory::Alloc(gpu0, sizeof(dst))); - - framework::DDim dst_dim({2, 2}); - framework::DDim dst_stride({2, 1}); - - platform::CUDADeviceContext ctx(gpu0); - TensorCopy(ctx, gpu_src + 1, src_stride, dst_dim, dst_stride, gpu_dst); - - memory::Copy(cpu, dst, gpu0, gpu_dst, sizeof(dst)); - - ASSERT_EQ(1, dst[0]); - ASSERT_EQ(2, dst[1]); - ASSERT_EQ(3, dst[2]); - ASSERT_EQ(4, dst[3]); - - memory::Free(gpu0, gpu_dst); - memory::Free(gpu0, gpu_src); -} - -#endif -} // namespace operators -} // namespace paddle \ No newline at end of file From 98ef17eddc6a691cd3a5bdaf8e1ab38d8d37f8cb Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Tue, 19 Sep 2017 22:28:16 -0700 Subject: [PATCH 20/35] move OpProtoAndCheckerMaker from operator to op_proto_maker --- paddle/framework/CMakeLists.txt | 4 +- paddle/framework/op_proto_maker.cc | 58 ++++++++++++++++ paddle/framework/op_proto_maker.h | 88 +++++++++++++++++++++++++ paddle/framework/op_proto_maker_test.cc | 51 ++++++++++++++ paddle/framework/op_registry.h | 1 + paddle/framework/operator.cc | 38 ----------- paddle/framework/operator.h | 65 ------------------ paddle/framework/operator_test.cc | 34 ---------- paddle/operators/prelu_op.h | 2 +- 9 files changed, 202 insertions(+), 139 deletions(-) create mode 100644 paddle/framework/op_proto_maker.cc create mode 100644 paddle/framework/op_proto_maker.h create mode 100644 paddle/framework/op_proto_maker_test.cc diff --git a/paddle/framework/CMakeLists.txt b/paddle/framework/CMakeLists.txt index 3371962c63..e535f84dba 100644 --- a/paddle/framework/CMakeLists.txt +++ b/paddle/framework/CMakeLists.txt @@ -19,12 +19,14 @@ cc_test(scope_test SRCS scope_test.cc DEPS scope) proto_library(framework_proto SRCS framework.proto) cc_library(attribute SRCS attribute.cc DEPS framework_proto) +cc_library(op_proto_maker SRCS op_proto_maker.cc DEPS framework_proto attribute) +cc_test(op_proto_maker_test SRCS op_proto_maker_test.cc DEPS op_proto_maker) cc_library(op_info SRCS op_info.cc DEPS attribute framework_proto) cc_library(operator SRCS operator.cc DEPS op_info device_context tensor scope) cc_test(operator_test SRCS operator_test.cc DEPS operator op_registry) cc_library(grad_op_builder SRCS grad_op_builder.cc DEPS operator) -cc_library(op_registry SRCS op_registry.cc DEPS grad_op_builder) +cc_library(op_registry SRCS op_registry.cc DEPS grad_op_builder op_proto_maker) cc_test(op_registry_test SRCS op_registry_test.cc DEPS op_registry) cc_test(grad_op_builder_test SRCS grad_op_builder_test.cc DEPS grad_op_builder op_registry add_op) diff --git a/paddle/framework/op_proto_maker.cc b/paddle/framework/op_proto_maker.cc new file mode 100644 index 0000000000..151d61d5b1 --- /dev/null +++ b/paddle/framework/op_proto_maker.cc @@ -0,0 +1,58 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/framework/op_proto_maker.h" + +namespace paddle { +namespace framework { + +void OpProtoAndCheckerMaker::Validate() { + validated_ = true; + CheckNoDuplicatedInOutAttrs(); +} + +OpProtoAndCheckerMaker::VariableBuilder OpProtoAndCheckerMaker::AddInput( + const std::string& name, const std::string& comment) { + auto* input = proto_->add_inputs(); + input->set_name(name); + input->set_comment(comment); + return OpProtoAndCheckerMaker::VariableBuilder{input}; +} + +OpProtoAndCheckerMaker::VariableBuilder OpProtoAndCheckerMaker::AddOutput( + const std::string& name, const std::string& comment) { + auto* output = proto_->add_outputs(); + output->set_name(name); + output->set_comment(comment); + return OpProtoAndCheckerMaker::VariableBuilder{output}; +} + +void OpProtoAndCheckerMaker::CheckNoDuplicatedInOutAttrs() { + std::unordered_set names; + auto checker = [&](const std::string& name) { + PADDLE_ENFORCE(!names.count(name), "[%s] is duplicated", name); + names.insert(name); + }; + for (auto& attr : proto_->attrs()) { + checker(attr.name()); + } + for (auto& input : proto_->inputs()) { + checker(input.name()); + } + for (auto& output : proto_->outputs()) { + checker(output.name()); + } +} + +} // namespace framework +} // namespace paddle diff --git a/paddle/framework/op_proto_maker.h b/paddle/framework/op_proto_maker.h new file mode 100644 index 0000000000..fea15a374b --- /dev/null +++ b/paddle/framework/op_proto_maker.h @@ -0,0 +1,88 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include "paddle/framework/attribute.h" +#include "paddle/framework/framework.pb.h" + +namespace paddle { +namespace framework { + +// this class not only make proto but also init attribute checkers. +class OpProtoAndCheckerMaker { + public: + OpProtoAndCheckerMaker(OpProto* proto, OpAttrChecker* op_checker) + : proto_(proto), op_checker_(op_checker) {} + + ~OpProtoAndCheckerMaker() { + PADDLE_ENFORCE(validated_, "should call Validate after build"); + } + + void Validate(); + + protected: + struct VariableBuilder { + OpProto::Var* var_; + + VariableBuilder& AsDuplicable() { + var_->set_duplicable(true); + return *this; + } + + VariableBuilder& AsIntermediate() { + var_->set_intermediate(true); + return *this; + } + + VariableBuilder& NotInGradient() { + var_->set_not_in_gradient(true); + return *this; + } + }; + + VariableBuilder AddInput(const std::string& name, const std::string& comment); + + VariableBuilder AddOutput(const std::string& name, + const std::string& comment); + + template + TypedAttrChecker& AddAttr(const std::string& name, + const std::string& comment, + bool generated = false) { + auto* attr = proto_->add_attrs(); + attr->set_name(name); + attr->set_comment(comment); + attr->set_generated(generated); + attr->set_type(AttrTypeID()); + return op_checker_->AddAttrChecker(name); + } + + void AddComment(const std::string& comment) { proto_->set_comment(comment); } + + private: + void CheckNoDuplicatedInOutAttrs(); + + OpProto* proto_; + OpAttrChecker* op_checker_; + bool validated_{false}; +}; + +class NOPMaker : public OpProtoAndCheckerMaker { + public: + NOPMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) {} +}; + +} // namespace framework +} // namespace paddle diff --git a/paddle/framework/op_proto_maker_test.cc b/paddle/framework/op_proto_maker_test.cc new file mode 100644 index 0000000000..b01e30f753 --- /dev/null +++ b/paddle/framework/op_proto_maker_test.cc @@ -0,0 +1,51 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/framework/op_proto_maker.h" + +#include "gtest/gtest.h" + +class TestAttrProtoMaker : public paddle::framework::OpProtoAndCheckerMaker { + public: + TestAttrProtoMaker(paddle::framework::OpProto* proto, + paddle::framework::OpAttrChecker* op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddAttr("scale", "scale of test op"); + AddAttr("scale", "scale of test op"); + } +}; + +TEST(ProtoMaker, DuplicatedAttr) { + paddle::framework::OpProto op_proto; + paddle::framework::OpAttrChecker op_checker; + auto proto_maker = TestAttrProtoMaker(&op_proto, &op_checker); + ASSERT_THROW(proto_maker.Validate(), paddle::platform::EnforceNotMet); +} + +class TestInOutProtoMaker : public paddle::framework::OpProtoAndCheckerMaker { + public: + TestInOutProtoMaker(paddle::framework::OpProto* proto, + paddle::framework::OpAttrChecker* op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("input", "input of test op"); + AddInput("input", "input of test op"); + } +}; + +TEST(ProtoMaker, DuplicatedInOut) { + paddle::framework::OpProto op_proto; + paddle::framework::OpAttrChecker op_checker; + auto proto_maker = TestInOutProtoMaker(&op_proto, &op_checker); + ASSERT_THROW(proto_maker.Validate(), paddle::platform::EnforceNotMet); +} \ No newline at end of file diff --git a/paddle/framework/op_registry.h b/paddle/framework/op_registry.h index 572dff860a..90077d0192 100644 --- a/paddle/framework/op_registry.h +++ b/paddle/framework/op_registry.h @@ -24,6 +24,7 @@ limitations under the License. */ #include "paddle/framework/framework.pb.h" #include "paddle/framework/grad_op_builder.h" #include "paddle/framework/op_info.h" +#include "paddle/framework/op_proto_maker.h" #include "paddle/framework/operator.h" #include "paddle/framework/scope.h" diff --git a/paddle/framework/operator.cc b/paddle/framework/operator.cc index f8a64a7866..49509af663 100644 --- a/paddle/framework/operator.cc +++ b/paddle/framework/operator.cc @@ -228,43 +228,5 @@ std::vector ExecutionContext::MultiOutput( return res; } -void OpProtoAndCheckerMaker::Validate() { - validated_ = true; - CheckNoDuplicatedInOutAttrs(); -} - -OpProtoAndCheckerMaker::VariableBuilder OpProtoAndCheckerMaker::AddInput( - const std::string& name, const std::string& comment) { - auto* input = proto_->add_inputs(); - input->set_name(name); - input->set_comment(comment); - return OpProtoAndCheckerMaker::VariableBuilder{input}; -} - -OpProtoAndCheckerMaker::VariableBuilder OpProtoAndCheckerMaker::AddOutput( - const std::string& name, const std::string& comment) { - auto* output = proto_->add_outputs(); - output->set_name(name); - output->set_comment(comment); - return OpProtoAndCheckerMaker::VariableBuilder{output}; -} - -void OpProtoAndCheckerMaker::CheckNoDuplicatedInOutAttrs() { - std::unordered_set names; - auto checker = [&](const std::string& name) { - PADDLE_ENFORCE(!names.count(name), "[%s] is duplicated", name); - names.insert(name); - }; - for (auto& attr : proto_->attrs()) { - checker(attr.name()); - } - for (auto& input : proto_->inputs()) { - checker(input.name()); - } - for (auto& output : proto_->outputs()) { - checker(output.name()); - } -} - } // namespace framework } // namespace paddle diff --git a/paddle/framework/operator.h b/paddle/framework/operator.h index b7c9c39402..1a78b6d1e1 100644 --- a/paddle/framework/operator.h +++ b/paddle/framework/operator.h @@ -167,71 +167,6 @@ class NOP : public OperatorBase { } }; -// this class not only make proto but also init attribute checkers. -class OpProtoAndCheckerMaker { - public: - OpProtoAndCheckerMaker(OpProto* proto, OpAttrChecker* op_checker) - : proto_(proto), op_checker_(op_checker) {} - - ~OpProtoAndCheckerMaker() { - PADDLE_ENFORCE(validated_, "should call Validate after build"); - } - - void Validate(); - - protected: - struct VariableBuilder { - OpProto::Var* var_; - - VariableBuilder& AsDuplicable() { - var_->set_duplicable(true); - return *this; - } - - VariableBuilder& AsIntermediate() { - var_->set_intermediate(true); - return *this; - } - - VariableBuilder& NotInGradient() { - var_->set_not_in_gradient(true); - return *this; - } - }; - - VariableBuilder AddInput(const std::string& name, const std::string& comment); - - VariableBuilder AddOutput(const std::string& name, - const std::string& comment); - - template - TypedAttrChecker& AddAttr(const std::string& name, - const std::string& comment, - bool generated = false) { - auto* attr = proto_->add_attrs(); - attr->set_name(name); - attr->set_comment(comment); - attr->set_generated(generated); - attr->set_type(AttrTypeID()); - return op_checker_->AddAttrChecker(name); - } - - void AddComment(const std::string& comment) { proto_->set_comment(comment); } - - private: - void CheckNoDuplicatedInOutAttrs(); - - OpProto* proto_; - OpAttrChecker* op_checker_; - bool validated_{false}; -}; - -class NOPMaker : public OpProtoAndCheckerMaker { - public: - NOPMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) - : OpProtoAndCheckerMaker(proto, op_checker) {} -}; - class InferShapeContext { public: InferShapeContext(const OperatorBase& op, const Scope& scope) diff --git a/paddle/framework/operator_test.cc b/paddle/framework/operator_test.cc index 20bbb11896..0beab0fac5 100644 --- a/paddle/framework/operator_test.cc +++ b/paddle/framework/operator_test.cc @@ -264,37 +264,3 @@ TEST(Operator, Clone) { auto b = a.Clone(); ASSERT_EQ(a.Type(), b->Type()); } - -class TestAttrProtoMaker : public paddle::framework::OpProtoAndCheckerMaker { - public: - TestAttrProtoMaker(paddle::framework::OpProto* proto, - paddle::framework::OpAttrChecker* op_checker) - : OpProtoAndCheckerMaker(proto, op_checker) { - AddAttr("scale", "scale of test op"); - AddAttr("scale", "scale of test op"); - } -}; - -TEST(ProtoMaker, DuplicatedAttr) { - paddle::framework::OpProto op_proto; - paddle::framework::OpAttrChecker op_checker; - auto proto_maker = TestAttrProtoMaker(&op_proto, &op_checker); - ASSERT_THROW(proto_maker.Validate(), paddle::platform::EnforceNotMet); -} - -class TestInOutProtoMaker : public paddle::framework::OpProtoAndCheckerMaker { - public: - TestInOutProtoMaker(paddle::framework::OpProto* proto, - paddle::framework::OpAttrChecker* op_checker) - : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("input", "input of test op"); - AddInput("input", "input of test op"); - } -}; - -TEST(ProtoMaker, DuplicatedInOut) { - paddle::framework::OpProto op_proto; - paddle::framework::OpAttrChecker op_checker; - auto proto_maker = TestInOutProtoMaker(&op_proto, &op_checker); - ASSERT_THROW(proto_maker.Validate(), paddle::platform::EnforceNotMet); -} \ No newline at end of file diff --git a/paddle/operators/prelu_op.h b/paddle/operators/prelu_op.h index 63031c25cc..9843c476e4 100644 --- a/paddle/operators/prelu_op.h +++ b/paddle/operators/prelu_op.h @@ -94,7 +94,7 @@ class PReluGradKernel : public framework::OpKernel { Transform(context.device_context(), out_ptr, out_ptr + numel, dout_ptr, dx_ptr, PReluGradFunctor(alpha_ptr)); - // TODO (Zhuoyuan): add dalpha upgrade when GPU kernels ready + // TODO(Zhuoyuan): add dalpha upgrade when GPU kernels ready } }; From c4ebd1e2aea3408df1b3692e1a4a404f7c937f2e Mon Sep 17 00:00:00 2001 From: Peng Li Date: Wed, 20 Sep 2017 14:41:16 +0800 Subject: [PATCH 21/35] Fix a few typos in docs --- doc/design/api.md | 4 +- doc/design/auto_gradient_check.md | 72 ++++++++++++------------ doc/design/functions_operators_layers.md | 4 +- doc/design/graph.md | 4 +- doc/design/parameters_in_cpp.md | 12 ++-- doc/design/reader/README.md | 2 +- doc/design/refactorization.md | 4 +- doc/design/releasing_process.md | 24 ++++---- doc/design/scope.md | 8 +-- doc/design/simple_op_design.md | 6 +- doc/design/var_desc.md | 2 +- paddle/framework/lod_tensor.md | 8 +-- 12 files changed, 75 insertions(+), 75 deletions(-) diff --git a/doc/design/api.md b/doc/design/api.md index 8185d2af0e..e6a4638d91 100644 --- a/doc/design/api.md +++ b/doc/design/api.md @@ -3,7 +3,7 @@ ## Ingredients As our design principle is starting from the essence: how could we -allow users to express and solve their problems at neural networks. +allow users to express and solve their problems as neural networks. Some essential concepts that our API have to provide include: 1. A *topology* is an expression of *layers*. @@ -233,7 +233,7 @@ paddle.dist_train(model, num_parameter_servers=15) ``` -The pseudo code if `paddle.dist_train` is as follows: +The pseudo code of `paddle.dist_train` is as follows: ```python def dist_train(topology, parameters, trainer, reader, ...): diff --git a/doc/design/auto_gradient_check.md b/doc/design/auto_gradient_check.md index 1f4d4ec16f..f9991541bc 100644 --- a/doc/design/auto_gradient_check.md +++ b/doc/design/auto_gradient_check.md @@ -1,17 +1,17 @@ ## Auto Gradient Checker Design ## Backgraound： -- Operator forward computing is easy to check if the result is right because it has a clear definition. **But** backpropagation is a notoriously difficult algorithm to debug and get right: - - 1. you should get the right backpropagation formula according to the forward computation. - - 2. you should implement it right in CPP. - - 3. it's difficult to prepare test data. +- Generally, it is easy to check whether the forward computation of an Operator is correct or not. However, backpropagation is a notoriously difficult algorithm to debug and get right: + 1. you should get the right backpropagation formula according to the forward computation. + 2. you should implement it right in CPP. + 3. it's difficult to prepare test data. -- Auto gradient check gets a numeric gradient by forward Operator and use it as a reference of the backward Operator's result. It has several advantages: - - 1. numeric gradient checker only need forward operator. - - 2. user only need to prepare the input data for forward Operator. +- Auto gradient checking gets a numerical gradient by forward Operator and use it as a reference of the backward Operator's result. It has several advantages: + 1. numerical gradient checker only need forward operator. + 2. user only need to prepare the input data for forward Operator. ## Mathematical Theory -The following two document from stanford has a detailed explanation of how to get numeric gradient and why it's useful. +The following two document from Stanford has a detailed explanation of how to get numerical gradient and why it's useful. - [Gradient checking and advanced optimization(en)](http://deeplearning.stanford.edu/wiki/index.php/Gradient_checking_and_advanced_optimization) - [Gradient checking and advanced optimization(cn)](http://ufldl.stanford.edu/wiki/index.php/%E6%A2%AF%E5%BA%A6%E6%A3%80%E9%AA%8C%E4%B8%8E%E9%AB%98%E7%BA%A7%E4%BC%98%E5%8C%96) @@ -20,7 +20,7 @@ The following two document from stanford has a detailed explanation of how to ge ## Numeric Gradient Implementation ### Python Interface ```python -def get_numeric_gradient(op, +def get_numerical_gradient(op, input_values, output_name, input_to_check, @@ -30,13 +30,13 @@ def get_numeric_gradient(op, Get Numeric Gradient for an operator's input. :param op: C++ operator instance, could be an network - :param input_values: The input variables. Should be an dictionary, key is - variable name. Value is numpy array. + :param input_values: The input variables. Should be an dictionary, whose key is + variable name, and value is numpy array. :param output_name: The final output variable name. - :param input_to_check: The input variable need to get gradient. + :param input_to_check: The input variable with respect to which to compute the gradient. :param delta: The perturbation value for numeric gradient method. The smaller delta is, the more accurate result will get. But if that delta is - too small, it could occur numerical stability problem. + too small, it will suffer from numerical stability problem. :param local_scope: The local scope used for get_numeric_gradient. :return: The gradient array in numpy format. """ @@ -45,28 +45,28 @@ def get_numeric_gradient(op, ### Explaination: - Why need `output_name` - - One Operator may have multiple Output, you can get independent gradient from each Output. So user should set one output to calculate. + - An Operator may have multiple Output, one can get independent gradient from each Output. So caller should specify the name of the output variable. - Why need `input_to_check` - - One operator may have multiple inputs. Gradient Op can calculate the gradient of these Inputs at the same time. But Numeric Gradient needs to calculate them one by one. So `get_numeric_gradient` is designed to calculate the gradient for one input. If you need to compute multiple inputs, you can call `get_numeric_gradient` multiple times. + - One operator may have multiple inputs. Gradient Op can calculate the gradient of these inputs at the same time. But Numeric Gradient needs to calculate them one by one. So `get_numeric_gradient` is designed to calculate the gradient for one input. If you need to compute multiple inputs, you can call `get_numeric_gradient` multiple times. ### Core Algorithm Implementation ```python - # we only compute gradient of one element each time. - # we use a for loop to compute the gradient of every element. + # we only compute gradient of one element a time. + # we use a for loop to compute the gradient of each element. for i in xrange(tensor_size): - # get one input element throw it's index i. + # get one input element by its index i. origin = tensor_to_check.get_float_element(i) - # add delta to it, run op and then get the sum of the result tensor. + # add delta to it, run op and then get the new value of the result tensor. x_pos = origin + delta tensor_to_check.set_float_element(i, x_pos) y_pos = get_output() - # plus delta to this element, run op and get the sum of the result tensor. + # plus delta to this element, run op and get the new value of the result tensor. x_neg = origin - delta tensor_to_check.set_float_element(i, x_neg) y_neg = get_output() @@ -85,15 +85,15 @@ def get_numeric_gradient(op, Each Operator Kernel has three kinds of Gradient: -- 1. Numeric Gradient -- 2. CPU Operator Gradient -- 3. GPU Operator Gradient(if supported) +1. Numerical gradient +2. CPU kernel gradient +3. GPU kernel gradient (if supported) -Numeric Gradient Only relies on forward Operator. So we use Numeric Gradient as the reference value. +The numerical gradient only relies on forward Operator. So we use the numerical gradient as the reference value. And the gradient checking is performed in the following three steps: -- 1. calculate the numeric gradient. -- 2. calculate CPU kernel Gradient with the backward Operator and compare it with the numeric gradient. -- 3. calculate GPU kernel Gradient with the backward Operator and compare it with the numeric gradient.(if support GPU) +1. calculate the numerical gradient +2. calculate CPU kernel gradient with the backward Operator and compare it with the numerical gradient +3. calculate GPU kernel gradient with the backward Operator and compare it with the numeric gradient (if supported) #### Python Interface @@ -110,8 +110,8 @@ Numeric Gradient Only relies on forward Operator. So we use Numeric Gradient as :param forward_op: used to create backward_op :param input_vars: numpy value of input variable. The following computation will use these variables. - :param inputs_to_check: inputs var names that should check gradient. - :param output_name: output name that used to + :param inputs_to_check: the input variable with respect to which to compute the gradient. + :param output_name: The final output variable name. :param max_relative_error: The relative tolerance parameter. :param no_grad_set: used when create backward ops :param only_cpu: only compute and check gradient on cpu kernel. @@ -120,24 +120,24 @@ Numeric Gradient Only relies on forward Operator. So we use Numeric Gradient as ``` ### How to check if two numpy array is close enough? -if `abs_numeric_grad` is nearly zero, then use abs error for numeric_grad, not relative +if `abs_numerical_grad` is nearly zero, then use abs error for numerical_grad ```python -numeric_grad = ... +numerical_grad = ... operator_grad = numpy.array(scope.find_var(grad_var_name(name)).get_tensor()) -abs_numeric_grad = numpy.abs(numeric_grad) -# if abs_numeric_grad is nearly zero, then use abs error for numeric_grad, not relative +abs_numerical_grad = numpy.abs(numerical_grad) +# if abs_numerical_grad is nearly zero, then use abs error for numeric_grad, not relative # error. -abs_numeric_grad[abs_numeric_grad < 1e-3] = 1 +abs_numerical_grad[abs_numerical_grad < 1e-3] = 1 -diff_mat = numpy.abs(abs_numeric_grad - operator_grad) / abs_numeric_grad +diff_mat = numpy.abs(abs_numerical_grad - operator_grad) / abs_numerical_grad max_diff = numpy.max(diff_mat) ``` #### Notes： -1，The Input data for auto gradient checker should be reasonable to avoid numeric problem. +The Input data for auto gradient checker should be reasonable to avoid numerical stability problem. #### Refs: diff --git a/doc/design/functions_operators_layers.md b/doc/design/functions_operators_layers.md index d23ba56b57..984b59f4c6 100644 --- a/doc/design/functions_operators_layers.md +++ b/doc/design/functions_operators_layers.md @@ -53,12 +53,12 @@ Let's explain using an example. Suppose that we are going to compose the FC usi ```python def operator.mul(X1, X2): O = Var() - paddle.cpp.create_operator("mul", input={X1, Y1], output=O) + paddle.cpp.create_operator("mul", input={X1, Y1}, output=O) return O def operator.add(X1, X2): O = Var() - paddle.cpp.create_operator("add", input={X1, X2], output=O) + paddle.cpp.create_operator("add", input={X1, X2}, output=O) return O ``` diff --git a/doc/design/graph.md b/doc/design/graph.md index 51b7f87638..7519a65df8 100644 --- a/doc/design/graph.md +++ b/doc/design/graph.md @@ -56,7 +56,7 @@ For each parameter, like W and b created by `layer.fc`, marked as double circles ## Block and Graph -The word block and graph are interchangable in the desgin of PaddlePaddle. A [Block[(https://github.com/PaddlePaddle/Paddle/pull/3708) is a metaphore of the code and local variables in a pair of curly braces in programming languages, where operators are like statements or instructions. A graph of operators and variables is a representation of the block. +The word block and graph are interchangable in the desgin of PaddlePaddle. A [Block](https://github.com/PaddlePaddle/Paddle/pull/3708) is a metaphore of the code and local variables in a pair of curly braces in programming languages, where operators are like statements or instructions. A graph of operators and variables is a representation of the block. A Block keeps operators in an array `BlockDesc::ops` @@ -67,4 +67,4 @@ message BlockDesc { } ``` -in the order that there appear in user programs, like the Python program at the beginning of this article. We can imagine that in `ops`, we have some forward operators, followed by some gradient operators, and then some optimization operators. +in the order that they appear in user programs, like the Python program at the beginning of this article. We can imagine that in `ops`, we have some forward operators, followed by some gradient operators, and then some optimization operators. diff --git a/doc/design/parameters_in_cpp.md b/doc/design/parameters_in_cpp.md index b6f99bc7d9..a7ac3f17c4 100644 --- a/doc/design/parameters_in_cpp.md +++ b/doc/design/parameters_in_cpp.md @@ -1,19 +1,19 @@ # Design Doc: The C++ Class `Parameters` -`Parameters` is a concept we designed in Paddle V2 API. `Parameters` is a container of parameters, and make Paddle can shared parameter between topologies. We described usages of `Parameter` in [api.md](./api.md). +`Parameters` is a concept we designed in PaddlePaddle V2 API. `Parameters` is a container of parameters, which makes PaddlePaddle capable of sharing parameter between topologies. We described usages of `Parameter` in [api.md](./api.md). -We used Python to implement Parameters when designing V2 API before. There are several defects for current implementation: +We used Python to implement Parameters when designing V2 API before. There are several defects for the current implementation: * We just use `memcpy` to share Parameters between topologies, but this is very inefficient. -* We did not implement share Parameters while training. We just trigger `memcpy` when start training. +* We did not support sharing Parameters while training. We just trigger `memcpy` when start training. -It is necessary that we implement Parameters in CPP side. However, it could be a code refactoring for Paddle, because Paddle was designed for training only one topology before, i.e., each GradientMachine contains its Parameter as a data member. In current Paddle implementation, there are three concepts associated with `Parameters`: +It is necessary that we implement Parameters in CPP side. However, it could result a code refactoring for PaddlePaddle, because PaddlePaddle was designed for training only one topology before, i.e., each GradientMachine contains its Parameter as a data member. In current PaddlePaddle implementation, there are three concepts associated with `Parameters`: 1. `paddle::Parameter`. A `Parameters` is a container for `paddle::Parameter`. It is evident that we should use `paddle::Parameter` when developing `Parameters`. However, the `Parameter` class contains many functions and does not have a clear interface. It contains `create/store Parameter`, `serialize/deserialize`, `optimize(i.e SGD)`, `randomize/zero`. When we developing `Parameters`, we only use `create/store Parameter` functionality. -We should extract functionalities of Parameter into many classes to clean Paddle CPP implementation. +We should extract functionalities of Parameter into many classes to clean PaddlePaddle CPP implementation. 2. `paddle::GradientMachine` and its sub-classes, e.g., `paddle::MultiGradientMachine`, `paddle::NeuralNetwork`. We should pass `Parameters` to `paddle::GradientMachine` when `forward/backward` to avoid `memcpy` between topologies. @@ -24,7 +24,7 @@ Also, we should handle multi-GPU/CPU training, because `forward` and `backward` So `Parameters` should be used by `paddle::ParameterUpdater`, and `paddle::ParameterUpdater` should optimize `Parameters` (by SGD). -The step by step approach for implementation Parameters in Paddle C++ core is listed below. Each step should be a PR and could be merged into Paddle one by one. +The step by step approach for implementation Parameters in PaddlePaddle C++ core is listed below. Each step should be a PR and could be merged into PaddlePaddle one by one. 1. Clean `paddle::Parameter` interface. Extract the functionalities of `paddle::Parameter` to prepare for the implementation of Parameters. diff --git a/doc/design/reader/README.md b/doc/design/reader/README.md index f21f7af520..320dccec3d 100644 --- a/doc/design/reader/README.md +++ b/doc/design/reader/README.md @@ -52,7 +52,7 @@ Here are valid outputs: # a mini batch of three data items, each data item is a list (single column). [([1,1,1],), ([2,2,2],), -([3,3,3],), +([3,3,3],)] ``` Please note that each item inside the list must be a tuple, below is an invalid output: diff --git a/doc/design/refactorization.md b/doc/design/refactorization.md index e105861e92..ad801ca421 100644 --- a/doc/design/refactorization.md +++ b/doc/design/refactorization.md @@ -15,7 +15,7 @@ The goal of refactorizaiton include: 1. Users write Python programs to describe the graphs and run it (locally or remotely). -1. A graph is composed of *variabels* and *operators*. +1. A graph is composed of *variables* and *operators*. 1. The description of graphs must be able to be serialized/deserialized, so it @@ -140,7 +140,7 @@ Compile Time -> IR -> Runtime * `thrust` has the same API as C++ standard library. Using `transform` can quickly implement a customized elementwise kernel. * `thrust` has more complex API, like `scan`, `reduce`, `reduce_by_key`. * Hand-writing `GPUKernel` and `CPU` code - * Do not write `.h`. CPU Kernel should be in `.cc`. CPU kernel should be in `.cu`. (`GCC` cannot compile GPU code.) + * Do not write `.h`. CPU Kernel should be in `.cc`. GPU kernel should be in `.cu`. (`GCC` cannot compile GPU code.) --- # Operator Register diff --git a/doc/design/releasing_process.md b/doc/design/releasing_process.md index 0c10e78280..62ff8f3229 100644 --- a/doc/design/releasing_process.md +++ b/doc/design/releasing_process.md @@ -1,8 +1,8 @@ -# Paddle发行规范 +# PaddlePaddle发行规范 -Paddle使用git-flow branching model做分支管理，使用[Semantic Versioning](http://semver.org/)标准表示Paddle版本号。 +PaddlePaddle使用git-flow branching model做分支管理，使用[Semantic Versioning](http://semver.org/)标准表示PaddlePaddle版本号。 -Paddle每次发新的版本，遵循以下流程: +PaddlePaddle每次发新的版本，遵循以下流程: 1. 从`develop`分支派生出新的分支，分支名为`release/版本号`。例如，`release/0.10.0` 2. 将新分支的版本打上tag，tag为`版本号rc.Patch号`。第一个tag为`0.10.0rc1`，第二个为`0.10.0rc2`，依次类推。 @@ -27,14 +27,14 @@ Paddle每次发新的版本，遵循以下流程: 需要注意的是: -* `release/版本号`分支一旦建立，一般不允许再从`develop`分支合入`release/版本号`。这样保证`release/版本号`分支功能的封闭，方便测试人员测试Paddle的行为。 +* `release/版本号`分支一旦建立，一般不允许再从`develop`分支合入`release/版本号`。这样保证`release/版本号`分支功能的封闭，方便测试人员测试PaddlePaddle的行为。 * 在`release/版本号`分支存在的时候，如果有bugfix的行为，需要将bugfix的分支同时merge到`master`, `develop`和`release/版本号`这三个分支。 -# Paddle 分支规范 +# PaddlePaddle 分支规范 -Paddle开发过程使用[git-flow](http://nvie.com/posts/a-successful-git-branching-model/)分支规范，并适应github的特性做了一些区别。 +PaddlePaddle开发过程使用[git-flow](http://nvie.com/posts/a-successful-git-branching-model/)分支规范，并适应github的特性做了一些区别。 -* Paddle的主版本库遵循[git-flow](http://nvie.com/posts/a-successful-git-branching-model/)分支规范。其中: +* PaddlePaddle的主版本库遵循[git-flow](http://nvie.com/posts/a-successful-git-branching-model/)分支规范。其中: * `master`分支为稳定(stable branch)版本分支。每一个`master`分支的版本都是经过单元测试和回归测试的版本。 * `develop`分支为开发(develop branch)版本分支。每一个`develop`分支的版本都经过单元测试，但并没有经过回归测试。 * `release/版本号`分支为每一次Release时建立的临时分支。在这个阶段的代码正在经历回归测试。 @@ -42,18 +42,18 @@ Paddle开发过程使用[git-flow](http://nvie.com/posts/a-successful-git-branch * 其他用户的fork版本库并不需要严格遵守[git-flow](http://nvie.com/posts/a-successful-git-branching-model/)分支规范，但所有fork的版本库的所有分支都相当于特性分支。 * 建议，开发者fork的版本库使用`develop`分支同步主版本库的`develop`分支 * 建议，开发者fork的版本库中，再基于`develop`版本fork出自己的功能分支。 - * 当功能分支开发完毕后，向Paddle的主版本库提交`Pull Reuqest`，进而进行代码评审。 + * 当功能分支开发完毕后，向PaddlePaddle的主版本库提交`Pull Reuqest`，进而进行代码评审。 * 在评审过程中，开发者修改自己的代码，可以继续在自己的功能分支提交代码。 * BugFix分支也是在开发者自己的fork版本库维护，与功能分支不同的是，BugFix分支需要分别给主版本库的`master`、`develop`与可能有的`release/版本号`分支，同时提起`Pull Request`。 -# Paddle回归测试列表 +# PaddlePaddle回归测试列表 -本列表说明Paddle发版之前需要测试的功能点。 +本列表说明PaddlePaddle发版之前需要测试的功能点。 -## Paddle Book中所有章节 +## PaddlePaddle Book中所有章节 -Paddle每次发版本首先要保证Paddle Book中所有章节功能的正确性。功能的正确性包括验证Paddle目前的`paddle_trainer`训练和纯使用`Python`训练模型正确性。 +PaddlePaddle每次发版本首先要保证PaddlePaddle Book中所有章节功能的正确性。功能的正确性包括验证PaddlePaddle目前的`paddle_trainer`训练和纯使用`Python`训练模型正确性。 | | 新手入门章节 | 识别数字 | 图像分类 | 词向量 | 情感分析 | 语意角色标注 | 机器翻译 | 个性化推荐 | | --- | --- | --- | --- | --- | --- | --- | --- | --- | diff --git a/doc/design/scope.md b/doc/design/scope.md index c9e0be716b..b1f9bb4378 100644 --- a/doc/design/scope.md +++ b/doc/design/scope.md @@ -17,7 +17,7 @@ Scope is an association of a name to variable. All variables belong to `Scope`. 1. Scope only contains a map of a name to variable. - All parameters, data, states in a Net should be variables and stored inside a scope. Each op should get inputs and outputs to do computation from a scope, such as data buffer, state(momentum) etc. + All parameters, data, states in a Net should be variables and stored inside a scope. Each op should get inputs and outputs to do computation from a scope, such as data buffer, state (momentum) etc. 1. Variable can only be created by Scope and a variable can only be got from Scope. User cannot create or get a variable outside a scope. This is a constraints of our framework, and will keep our framework simple and clear. @@ -32,7 +32,7 @@ Scope is an association of a name to variable. All variables belong to `Scope`. 1. Scope should destruct all Variables inside it when itself is destructed. User can never store `Variable` pointer somewhere else. - Because Variable can only be got from Scope. When destroying Scope, we also need to destroy all the Variables in it. If user store `Variable` pointer to private data member or some global variable, the pointer will be a invalid pointer when associated `Scope` is destroyed. + Because Variable can only be got from Scope. When destroying Scope, we also need to destroy all the Variables in it. If user store `Variable` pointer to private data member or some global variable, the pointer will be an invalid pointer when associated `Scope` is destroyed. ```cpp class Scope { @@ -50,7 +50,7 @@ class Scope { Just like [scope](https://en.wikipedia.org/wiki/Scope_(computer_science)) in programming languages, `Scope` in the neural network can also be a local scope. There are two attributes about local scope. -1. We can create local variables in a local scope. When that local scope are destroyed, all local variables should also be destroyed. +1. We can create local variables in a local scope. When that local scope is destroyed, all local variables should also be destroyed. 2. Variables in a parent scope can be retrieved from local scopes of that parent scope, i.e., when user get a variable from a scope, it will try to search this variable in current scope. If there is no such variable in the local scope, `scope` will keep searching from its parent, until the variable is found or there is no parent. ```cpp @@ -121,4 +121,4 @@ Also, as the parent scope is a `shared_ptr`, we can only `Create()` a scope shar ## Orthogonal interface -`FindVar` will return `nullptr` when `name` is not found. It can be used as `Contains` method. `NewVar` will return a `Error` when there is a name conflict locally. Combine `FindVar` and `NewVar`, we can implement `NewVar` easily. +`FindVar` will return `nullptr` when `name` is not found. It can be used as `Contains` method. `NewVar` will return an `Error` when there is a name conflict locally. Combine `FindVar` and `NewVar`, we can implement `NewVar` easily. diff --git a/doc/design/simple_op_design.md b/doc/design/simple_op_design.md index fded4a6861..c7aeed7f9b 100644 --- a/doc/design/simple_op_design.md +++ b/doc/design/simple_op_design.md @@ -6,9 +6,9 @@ The Interaction between Python and C++ can be simplified as two steps: 1. C++ tells Python how many Ops there are, and what parameter do users need to offer to initialize a new Op. Python then builds API for each Op at compile time. -2. Users invoke APIs built by Python and provide necessary parameters. These parameters will be sent to C++ fo finish Op construction task. +2. Users invoke APIs built by Python and provide necessary parameters. These parameters will be sent to C++ for finishing the Op construction task. -### Message form C++ to Python +### Message from C++ to Python We define a Protobuf message class `OpProto` to hold message needed in the first step. What should an `OpProto` contain? This question is equivalent to “What message do we need to offer, to build a Python API which is legal and user oriented and can use to describe a whole Op.” @@ -193,7 +193,7 @@ def fc_layer(input, size, with_bias, activation): elif: # ... return act_output; -``` +``` ### Low Leval API diff --git a/doc/design/var_desc.md b/doc/design/var_desc.md index 86a95c10d5..bfbbdd0578 100644 --- a/doc/design/var_desc.md +++ b/doc/design/var_desc.md @@ -1,7 +1,7 @@ ## Background PaddlePaddle divides the description of neural network computation graph into two stages: compile time and runtime. -PaddlePaddle use proto message to describe compile time graph for +PaddlePaddle use proto message to describe compile time graph because 1. Computation graph should be able to be saved to a file. 1. In distributed training, the graph will be serialized and send to multiple workers. diff --git a/paddle/framework/lod_tensor.md b/paddle/framework/lod_tensor.md index 769b61f175..b824de393a 100644 --- a/paddle/framework/lod_tensor.md +++ b/paddle/framework/lod_tensor.md @@ -4,7 +4,7 @@ PaddlePaddle's RNN doesn't require that all instances have the same length. To ## Challenge of Variable-length Inputs -People usually represent a mini-batch by a Tensor. For example, a mini-batch of 32 images, each of size 32x32, is a 10x32x32 Tensor. So a transformation, T, of all images can be a matrix multiplication of the 32x32xO-dimensional tensor T and the 10x32x32 Tensor. +People usually represent a mini-batch by a Tensor. For example, a mini-batch of 10 images, each of size 32x32, is a 10x32x32 Tensor. So a transformation, T, of all images can be a matrix multiplication of the 10xOx32-dimensional tensor T and the 10x32x32 Tensor. Another example is that each mini-batch contains 32 sentences, where each word is a D-dimensional one-hot vector. If all sentences have the same length L, we can represent this mini-batch by a 32xLxD tensor. However, in most cases, sentences have variable lengths, and we will need an index data structure to record these variable lengths. @@ -54,7 +54,7 @@ In summary, as long as that the essential elements (words or images) have the s - the first dimension size L has an additon property -- a LoD index as a nested vector: ```c++ - typedef std::vector > LoD; + typedef std::vector> LoD; ``` - The LoD index can is not necessary when there are only two levels and all elements of the second level have length 1. @@ -99,7 +99,7 @@ Let's go on slicing this slice. Its <1,1>-slice is The algorithm, with over-simplified data structure, is defined as ```c++ -typedef vector > LoD; +typedef std::vector> LoD; struct LoDTensor { LoD lod_; @@ -128,7 +128,7 @@ Suppose that we want to retrieve the <1,2>-slice we will need to find out the starting position of this slice by summing over all leaf nodes in `LoD` to the left of the slice, i.e., 3 + 2 + 4 + 1 = 10. -To avoid the traversal of the LoD tree at slcing time, we can do it at the construction time -- instead of saving the lengths of the next level in the LoD tree, we can save the starting offset of the next level. For example, above LoD Tensor can be transformed into +To avoid the traversal of the LoD tree at slicing time, we can do it at the construction time -- instead of saving the lengths of the next level in the LoD tree, we can save the starting offset of the next level. For example, above LoD Tensor can be transformed into ``` 0 From e635e3fd2be4313b2e2e1491f54f95468899ebe7 Mon Sep 17 00:00:00 2001 From: ranqiu Date: Wed, 20 Sep 2017 14:55:49 +0800 Subject: [PATCH 22/35] Update faq of the doc --- doc/faq/index_cn.rst | 49 ++++++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/doc/faq/index_cn.rst b/doc/faq/index_cn.rst index 119e037aa2..3a7dc4a93c 100644 --- a/doc/faq/index_cn.rst +++ b/doc/faq/index_cn.rst @@ -322,33 +322,12 @@ pip uninstall py_paddle paddle pip install python/dist/paddle*.whl && pip install ../paddle/dist/py_paddle*.whl -16. 如何加载预训练embedding参数 ------------------------------- - -设置embedding的参数属性 :code:`is_static=True`，使embedding参数在训练过程中保持不变，在创建parameters后，使用 :code:`parameters.set()` 加载预训练参数。 - -.. code-block:: python - - def load_parameter(file_name, h, w): - with open(file_name, 'rb') as f: - f.read(16) # skip header. - return np.fromfile(f, dtype=np.float32).reshape(h, w) - - - emb_para = paddle.attr.Param(name='emb', initial_std=0., is_static=True) - paddle.layer.embedding(size=word_dim, input=x, param_attr=emb_para) - - - parameters = paddle.parameters.create(my_cost) - parameters.set('emb', load_parameter(emb_param_file, 30000, 256)) - - -17. PaddlePaddle存储的参数格式是什么，如何和明文进行相互转化 +16. PaddlePaddle存储的参数格式是什么，如何和明文进行相互转化 --------------------------------------------------------- -PaddlePaddle保存的二进制参数文件内容由16位头信息和网络参数两部分组成。头信息中，第一位固定为0，第二位为4，在使用double精度时，第二位为8，第三位记录共有多少个数值。 +PaddlePaddle保存的模型参数文件内容由16字节头信息和网络参数两部分组成。头信息中，1~4字节表示PaddlePaddle版本信息；5~8字节表示每个参数占用的字节数，当保存的网络参数为float类型时为4，double类型时为8；9~16字节表示保存的参数总个数。 -将PaddlePaddle保存的二进制参数还原回明文时，先跳过PaddlePaddle模型参数文件的头信息，再提取网络参数，示例如下： +将PaddlePaddle保存的模型参数还原回明文时，可以使用相应数据类型的 :code:`numpy.array` 加载具体网络参数，此时需要跳过PaddlePaddle模型参数文件的头信息。一般情况下，PaddlePaddle保存的模型参数数据类型为float，所以在使用 :code:`numpy.array` 时一般设置 :code:`dtype=float32` 。示例如下： .. code-block:: python @@ -361,7 +340,7 @@ PaddlePaddle保存的二进制参数文件内容由16位头信息和网络参数 fmt="%.6f", delimiter=",") -将明文参数转化为PaddlePaddle可加载的模型参数时，先根据参数规模写入头信息，再写入具体网络参数。以下为将随机生成的矩阵转化为PaddlePaddle可加载的模型参数示例: +将明文参数转化为PaddlePaddle可加载的模型参数时，先根据数据类型和参数规模写入头信息，再写入具体网络参数。以下为将随机生成的矩阵转化为PaddlePaddle可加载的模型参数示例: .. code-block:: python @@ -371,3 +350,23 @@ PaddlePaddle保存的二进制参数文件内容由16位头信息和网络参数 param = np.float32(np.random.rand(height, width)) with open(param_file, "w") as fparam: fparam.write(header + param.tostring()) + +17. 如何加载预训练embedding参数 +------------------------------ + +设置embedding的参数属性 :code:`is_static=True`，使embedding参数在训练过程中保持不变，从模型文件将预训练参数载入 :code:`numpy.array`，在创建parameters后，使用 :code:`parameters.set()` 加载预训练参数。PaddlePaddle保存的模型参数文件前16字节为头信息，用户将参数载入 :code:`numpy.array` 时须从第17字节开始。 + +.. code-block:: python + + def load_parameter(file_name, h, w): + with open(file_name, 'rb') as f: + f.read(16) # skip header. + return np.fromfile(f, dtype=np.float32).reshape(h, w) + + + emb_para = paddle.attr.Param(name='emb', initial_std=0., is_static=True) + paddle.layer.embedding(size=word_dim, input=x, param_attr=emb_para) + + + parameters = paddle.parameters.create(my_cost) + parameters.set('emb', load_parameter(emb_param_file, 30000, 256)) From 69c613fb5af7ff5b289fad7525d01a2dce37f749 Mon Sep 17 00:00:00 2001 From: Luo Tao Date: Wed, 20 Sep 2017 14:08:24 +0800 Subject: [PATCH 23/35] refine Layer.cpp for some CostLayer --- paddle/gserver/layers/Layer.cpp | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/paddle/gserver/layers/Layer.cpp b/paddle/gserver/layers/Layer.cpp index f11875b176..e95f42c863 100644 --- a/paddle/gserver/layers/Layer.cpp +++ b/paddle/gserver/layers/Layer.cpp @@ -14,7 +14,8 @@ limitations under the License. */ #include "paddle/utils/Util.h" -#include "Layer.h" +#include "CostLayer.h" +#include "ValidationLayer.h" #include "paddle/math/SparseMatrix.h" #include "paddle/utils/Error.h" #include "paddle/utils/Logging.h" @@ -93,6 +94,20 @@ ClassRegistrar Layer::registrar_; LayerPtr Layer::create(const LayerConfig& config) { std::string type = config.type(); + + // NOTE: As following types have illegal character '-', + // they can not use REGISTER_LAYER to registrar. + // Besides, to fit with old training models, + // they can not use '_' instead. + if (type == "multi-class-cross-entropy") + return LayerPtr(new MultiClassCrossEntropy(config)); + else if (type == "rank-cost") + return LayerPtr(new RankingCost(config)); + else if (type == "auc-validation") + return LayerPtr(new AucValidation(config)); + else if (type == "pnpair-validation") + return LayerPtr(new PnpairValidation(config)); + return LayerPtr(registrar_.createByType(config.type(), config)); } From cabd643438418dda2b14315a8f30f5f7246e3796 Mon Sep 17 00:00:00 2001 From: ranqiu Date: Wed, 20 Sep 2017 15:58:03 +0800 Subject: [PATCH 24/35] Update faq of doc --- doc/faq/index_cn.rst | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/doc/faq/index_cn.rst b/doc/faq/index_cn.rst index 3a7dc4a93c..209dac6989 100644 --- a/doc/faq/index_cn.rst +++ b/doc/faq/index_cn.rst @@ -325,9 +325,9 @@ pip install python/dist/paddle*.whl && pip install ../paddle/dist/py_paddle*.whl 16. PaddlePaddle存储的参数格式是什么，如何和明文进行相互转化 --------------------------------------------------------- -PaddlePaddle保存的模型参数文件内容由16字节头信息和网络参数两部分组成。头信息中，1~4字节表示PaddlePaddle版本信息；5~8字节表示每个参数占用的字节数，当保存的网络参数为float类型时为4，double类型时为8；9~16字节表示保存的参数总个数。 +PaddlePaddle保存的模型参数文件内容由16字节头信息和网络参数两部分组成。头信息中，1~4字节表示PaddlePaddle版本信息，在多数情况下，可以直接填充0；5~8字节表示每个参数占用的字节数，当保存的网络参数为float类型时为4，double类型时为8；9~16字节表示保存的参数总个数。 -将PaddlePaddle保存的模型参数还原回明文时，可以使用相应数据类型的 :code:`numpy.array` 加载具体网络参数，此时需要跳过PaddlePaddle模型参数文件的头信息。一般情况下，PaddlePaddle保存的模型参数数据类型为float，所以在使用 :code:`numpy.array` 时一般设置 :code:`dtype=float32` 。示例如下： +将PaddlePaddle保存的模型参数还原回明文时，可以使用相应数据类型的 :code:`numpy.array` 加载具体网络参数，此时可以跳过PaddlePaddle模型参数文件的头信息。若在PaddlePaddle编译时，未指定按照double精度编译，默认情况下按照float精度计算，保存的参数也是float类型。这时在使用 :code:`numpy.array` 时，一般设置 :code:`dtype=float32` 。示例如下： .. code-block:: python @@ -340,7 +340,7 @@ PaddlePaddle保存的模型参数文件内容由16字节头信息和网络参数 fmt="%.6f", delimiter=",") -将明文参数转化为PaddlePaddle可加载的模型参数时，先根据数据类型和参数规模写入头信息，再写入具体网络参数。以下为将随机生成的矩阵转化为PaddlePaddle可加载的模型参数示例: +将明文参数转化为PaddlePaddle可加载的模型参数时，首先构造头信息，再写入网络参数。下面将随机生成的矩阵转化为可以被PaddlePaddle加载的模型参数。 .. code-block:: python @@ -351,10 +351,18 @@ PaddlePaddle保存的模型参数文件内容由16字节头信息和网络参数 with open(param_file, "w") as fparam: fparam.write(header + param.tostring()) -17. 如何加载预训练embedding参数 +17. 如何加载预训练参数 ------------------------------ -设置embedding的参数属性 :code:`is_static=True`，使embedding参数在训练过程中保持不变，从模型文件将预训练参数载入 :code:`numpy.array`，在创建parameters后，使用 :code:`parameters.set()` 加载预训练参数。PaddlePaddle保存的模型参数文件前16字节为头信息，用户将参数载入 :code:`numpy.array` 时须从第17字节开始。 +* 对加载预训练参数的层，设置其参数属性 :code:`is_static=True`，使该层的参数在训练过程中保持不变。以embedding层为例，代码如下： + +.. code-block:: python + + emb_para = paddle.attr.Param(name='emb', is_static=True) + paddle.layer.embedding(size=word_dim, input=x, param_attr=emb_para) + + +* 从模型文件将预训练参数载入 :code:`numpy.array`，在创建parameters后，使用 :code:`parameters.set()` 加载预训练参数。PaddlePaddle保存的模型参数文件前16字节为头信息，用户将参数载入 :code:`numpy.array` 时须从第17字节开始。以embedding层为例，代码如下： .. code-block:: python @@ -363,10 +371,5 @@ PaddlePaddle保存的模型参数文件内容由16字节头信息和网络参数 f.read(16) # skip header. return np.fromfile(f, dtype=np.float32).reshape(h, w) - - emb_para = paddle.attr.Param(name='emb', initial_std=0., is_static=True) - paddle.layer.embedding(size=word_dim, input=x, param_attr=emb_para) - - parameters = paddle.parameters.create(my_cost) parameters.set('emb', load_parameter(emb_param_file, 30000, 256)) From 4e3ba65f193b01a4b514f5cf0ba975cd35beb41e Mon Sep 17 00:00:00 2001 From: yangyaming Date: Wed, 20 Sep 2017 15:59:33 +0800 Subject: [PATCH 25/35] Refine doc. --- paddle/operators/smooth_l1_loss_op.cc | 63 +++++++++++-------- paddle/operators/smooth_l1_loss_op.h | 4 +- .../framework/tests/test_smooth_l1_loss_op.py | 14 ++--- 3 files changed, 46 insertions(+), 35 deletions(-) diff --git a/paddle/operators/smooth_l1_loss_op.cc b/paddle/operators/smooth_l1_loss_op.cc index 427ca96d1f..9ee6fff8db 100644 --- a/paddle/operators/smooth_l1_loss_op.cc +++ b/paddle/operators/smooth_l1_loss_op.cc @@ -23,19 +23,15 @@ class SmoothL1LossOp : public framework::OperatorWithKernel { protected: void InferShape(const framework::InferShapeContext& ctx) const override { - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), - "Input of SmoothL1LossOp must be initialized."); - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Y"), - "Target of SmoothL1LossOp must be initialized."); + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), "X must be initialized."); + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Y"), "Y must be initialized."); auto* x = ctx.Input("X"); auto* y = ctx.Input("Y"); PADDLE_ENFORCE_EQ(x->dims(), y->dims(), - "Dimensions of SmoothL1LossOp's input and target " - "must be same."); + "The shape of X and Y must be the same."); PADDLE_ENFORCE_GE(x->dims().size(), 2, - "Tensor rank of SmoothL1LossOp's input must be " - "at least 2."); + "The tensor rank of X must be at least 2."); auto* inside_weight = ctx.Input("InsideWeight"); if (inside_weight) { auto* outside_weight = ctx.Input("OutsideWeight"); @@ -43,10 +39,9 @@ class SmoothL1LossOp : public framework::OperatorWithKernel { "If weights are provided, must specify both " "inside and outside weights."); PADDLE_ENFORCE_EQ(inside_weight->dims(), x->dims(), - "Dimensions of inside weight must be same with input."); - PADDLE_ENFORCE_EQ( - outside_weight->dims(), x->dims(), - "Dimensions of outside weight must be same with input."); + "The shape of InsideWeight must be same as X."); + PADDLE_ENFORCE_EQ(outside_weight->dims(), x->dims(), + "The shape of OutsideWeight must be same as X."); } auto* diff = ctx.Output("Diff"); @@ -63,21 +58,37 @@ class SmoothL1LossOpMaker : public framework::OpProtoAndCheckerMaker { SmoothL1LossOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("X", "Input of SmoothL1LossOp."); - AddInput("Y", "Target of SmoothL1LossOp."); - AddInput("InsideWeight", "Optional input to scale (X-Y)."); - AddInput("OutsideWeight", "Optinal input to scale smooth l1 loss."); - AddOutput("Diff", "Intermediate variable to cache Win*(X-Y).") + AddInput("X", + "The input tensor of smooth l1 loss op." + "The rank should be greater or equal to 2 with shape " + "[batch_size, value_dim1, value_dim2, ..., value_dimN]"); + AddInput("Y", + "The target tensor of smooth l1 loss op " + "with the same shape as X."); + AddInput("InsideWeight", + "Optional input tensor of smooth l1 loss op with the same shape " + "as X. If provided, the result of (X - Y) will be multiplied " + "by this tensor element by element."); + AddInput("OutsideWeight", + "Optinal input of smooth l1 loss op with the same shape as X." + "If provided, the output smooth l1 loss will be multiplied by " + "this tensor element by element."); + AddOutput("Diff", "Intermediate variable to cache InsideWeight*(X-Y).") .AsIntermediate(); - AddOutput("Out", "Final smooth l1 loss of inputs."); - AddAttr("sigma", "Hyper parameter, default value is 3.0 .") + AddOutput("Out", "Smooth l1 loss."); + AddAttr("sigma", + "Hyper parameter of smooth l1 loss op." + "A float scalar with default value 3.0.") .SetDefault(3.0); AddComment(R"DOC( -Compute SmoothL1Loss for input and target. +Compute smooth l1 loss for input and target. The operator take the 1st +dimension of input as batch size. For each instance, it will compute +smooth l1 loss element by element first and sum all losses to one value. +So the output shape is [batch_size, 1]. The equation is: -loss = 0.5 * (sigma * (x - y)) ^ 2 if abs(x - y) < 1 / sigma^2 - abs(x - y) - 0.5 / sigma^2 otherwise +loss = 0.5 * (sigma * (x-y))^2 if abs(x - y) < 1 / sigma^2 + abs(x - y) - 0.5 / sigma^2 otherwise )DOC"); } @@ -98,12 +109,12 @@ class SmoothL1LossGradOp : public framework::OperatorWithKernel { ctx.Output(framework::GradVarName("Y")); PADDLE_ENFORCE_GE(out_dims.size(), 2, - "Tensor rank of output gradient should be 2."); + "The tensor rank of Input(Out@Grad) should be 2."); PADDLE_ENFORCE_EQ(out_dims[0], in_dims[0], - "First dimension of ouptut gradient must be " - "same with input."); + "The 1st dimension of Input(Out@Grad) must be " + "same as input."); PADDLE_ENFORCE_EQ(out_dims[1], 1, - "Second dimension of output gradient must be 1."); + "The 2nd dimension of Input(Out@Grad) must be 1."); if (x_grad) x_grad->Resize(in_dims); if (y_grad) y_grad->Resize(in_dims); diff --git a/paddle/operators/smooth_l1_loss_op.h b/paddle/operators/smooth_l1_loss_op.h index 90f23f5a0c..0604fb5e1c 100644 --- a/paddle/operators/smooth_l1_loss_op.h +++ b/paddle/operators/smooth_l1_loss_op.h @@ -59,7 +59,7 @@ class SmoothL1LossKernel : public framework::OpKernel { out1->mutable_data(context.GetPlace()); auto place = context.GetEigenDevice(); - auto sigma = static_cast(context.op().Attr("sigma")); + auto sigma = static_cast(context.Attr("sigma")); T sigma2 = sigma * sigma; bool has_weight = (in2 != nullptr) && (in3 != nullptr); @@ -122,7 +122,7 @@ class SmoothL1LossGradKernel : public framework::OpKernel { auto* in1 = context.Input("OutsideWeight"); auto* in2 = context.Input("Diff"); auto* og = context.Input(framework::GradVarName("Out")); - auto sigma = static_cast(context.op().Attr("sigma")); + auto sigma = static_cast(context.Attr("sigma")); T sigma2 = sigma * sigma; bool has_weight = (in0 != nullptr) && (in1 != nullptr); diff --git a/python/paddle/v2/framework/tests/test_smooth_l1_loss_op.py b/python/paddle/v2/framework/tests/test_smooth_l1_loss_op.py index 1b79f16abe..be940327ec 100644 --- a/python/paddle/v2/framework/tests/test_smooth_l1_loss_op.py +++ b/python/paddle/v2/framework/tests/test_smooth_l1_loss_op.py @@ -14,7 +14,7 @@ def smooth_l1_loss_forward(val, sigma2): class TestSmoothL1LossOp1(OpTest): def setUp(self): self.op_type = "smooth_l1_loss" - dims = (6, 10) + dims = (5, 10) self.inputs = { 'X': np.random.random(dims).astype("float32"), 'Y': np.random.random(dims).astype("float32") @@ -35,17 +35,17 @@ class TestSmoothL1LossOp1(OpTest): def test_check_grad_ingore_x(self): self.check_grad( - ['Y'], 'Out', max_relative_error=0.02, no_grad_set=set("X")) + ['Y'], 'Out', max_relative_error=0.03, no_grad_set=set("X")) def test_check_grad_ingore_y(self): self.check_grad( - ['X'], 'Out', max_relative_error=0.02, no_grad_set=set('Y')) + ['X'], 'Out', max_relative_error=0.03, no_grad_set=set('Y')) class TestSmoothL1LossOp2(OpTest): def setUp(self): self.op_type = "smooth_l1_loss" - dims = (6, 10) + dims = (5, 10) self.inputs = { 'X': np.random.random(dims).astype("float32"), 'Y': np.random.random(dims).astype("float32"), @@ -66,20 +66,20 @@ class TestSmoothL1LossOp2(OpTest): self.check_output() def test_check_grad_normal(self): - self.check_grad(['X', 'Y'], 'Out', max_relative_error=0.02) + self.check_grad(['X', 'Y'], 'Out', max_relative_error=0.03) def test_check_grad_ingore_x(self): self.check_grad( ['Y'], 'Out', - max_relative_error=0.02, + max_relative_error=0.03, no_grad_set=set(['X', 'InsideWeight', 'OutsideWeight'])) def test_check_grad_ingore_y(self): self.check_grad( ['X'], 'Out', - max_relative_error=0.02, + max_relative_error=0.03, no_grad_set=set(['Y', 'InsideWeight', 'OutsideWeight'])) From fa722385def69cf2cb08a6570592360516eafde1 Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Wed, 20 Sep 2017 14:54:11 +0800 Subject: [PATCH 26/35] refine test_MKLDNN and skip memory copy for relu --- paddle/gserver/activations/MKLDNNActivation.h | 5 +- paddle/gserver/tests/test_MKLDNN.cpp | 127 +++++++++--------- 2 files changed, 65 insertions(+), 67 deletions(-) diff --git a/paddle/gserver/activations/MKLDNNActivation.h b/paddle/gserver/activations/MKLDNNActivation.h index bda9bbebe5..86ffe38736 100644 --- a/paddle/gserver/activations/MKLDNNActivation.h +++ b/paddle/gserver/activations/MKLDNNActivation.h @@ -131,8 +131,9 @@ public: fwdPD_.reset(new eltwise_fwd::primitive_desc(fwdDesc, eng)); // use inplace for forward but save input value before submit inVal_ = val_; - if (act.grad) { - // only copy when need do backward + copyInVal_ = nullptr; + if (act.grad && algo == mkldnn::algorithm::eltwise_tanh) { + // tanh need save src input for backward inVal_ = MKLDNNMatrix::create(nullptr, val_->getPrimitiveDesc()); copyInVal_ = std::make_shared(*val_, *inVal_); CHECK(copyInVal_) << "should not be emptry"; diff --git a/paddle/gserver/tests/test_MKLDNN.cpp b/paddle/gserver/tests/test_MKLDNN.cpp index 406181370f..1bfbbde424 100644 --- a/paddle/gserver/tests/test_MKLDNN.cpp +++ b/paddle/gserver/tests/test_MKLDNN.cpp @@ -26,17 +26,26 @@ DECLARE_bool(thread_local_rand_use_global_seed); DECLARE_bool(use_gpu); DECLARE_bool(use_mkldnn); -struct testFCDesc { +#define RUN_MKLDNN_TEST(DNN_CONFIG, REF_CONFIG, DESC) \ + MKLDNNTester tester; \ + for (auto bs : {DESC.bs, 1}) { \ + tester.run(DNN_CONFIG, REF_CONFIG, bs, DESC.ih, DESC.iw); \ + } + +#define RUN_MKLDNN_TEST_LAYER(DNN_CONFIG, REF_TYPE, DESC) \ + TestConfig ref = DNN_CONFIG; \ + ref.layerConfig.set_type(REF_TYPE); \ + RUN_MKLDNN_TEST(DNN_CONFIG, ref, DESC) + +struct testFcDesc { int bs; int ic; int oc; int ih, iw; // oh == ow == 1 }; -void testFcLayer(const testFCDesc& pm) { - const std::string compareTypes[] = {"mkldnn_fc", "fc"}; - TestConfig cfg; - cfg.layerConfig.set_type(compareTypes[0]); +static void getMKLDNNFcConfig(TestConfig& cfg, const testFcDesc& pm) { + cfg.layerConfig.set_type("mkldnn_fc"); cfg.layerConfig.set_size(pm.oc); cfg.inputDefs.push_back( {INPUT_DATA, @@ -44,25 +53,25 @@ void testFcLayer(const testFCDesc& pm) { /* size of input layer= */ size_t(pm.ic * pm.ih * pm.iw), /* size of weight= */ size_t(pm.oc * pm.ic * pm.ih * pm.iw)}); cfg.layerConfig.add_inputs(); +} - MKLDNNTester tester; +void testFcLayer(const testFcDesc& pm) { + TestConfig dnnConfig; + getMKLDNNFcConfig(dnnConfig, pm); for (auto biasSize : {pm.oc, 0}) { - cfg.biasSize = biasSize; - TestConfig ref = cfg; - ref.layerConfig.set_type(compareTypes[1]); - for (auto bs : {pm.bs, 1}) { - tester.run(cfg, ref, bs, pm.ih, pm.iw); - } + dnnConfig.biasSize = biasSize; + RUN_MKLDNN_TEST_LAYER(dnnConfig, "fc", pm) } } TEST(MKLDNNLayer, FcLayer) { - testFcLayer({/*bs*/ 2, /*ic*/ 2, /*oc*/ 3, /*ih*/ 1, /*iw*/ 1}); - testFcLayer({/*bs*/ 3, /*ic*/ 7, /*oc*/ 19, /*ih*/ 1, /*iw*/ 1}); - testFcLayer({/*bs*/ 8, /*ic*/ 16, /*oc*/ 32, /*ih*/ 13, /*iw*/ 13}); - testFcLayer({/*bs*/ 4, /*ic*/ 12, /*oc*/ 18, /*ih*/ 13, /*iw*/ 11}); - testFcLayer({/*bs*/ 2, /*ic*/ 64, /*oc*/ 32, /*ih*/ 16, /*iw*/ 16}); - testFcLayer({/*bs*/ 15, /*ic*/ 3, /*oc*/ 6, /*ih*/ 16, /*iw*/ 16}); + /* bs, ic, ih, iw, oc */ + testFcLayer({2, 2, 1, 1, 3}); + testFcLayer({3, 7, 1, 1, 19}); + testFcLayer({8, 16, 13, 13, 32}); + testFcLayer({4, 12, 13, 13, 18}); + testFcLayer({2, 64, 16, 16, 32}); + testFcLayer({15, 3, 16, 16, 6}); } struct testConvDesc { @@ -75,13 +84,10 @@ struct testConvDesc { int dh, dw; }; -void testConvLayer(const testConvDesc& pm) { - const std::string compareTypes[] = {"mkldnn_conv", "exconv"}; - TestConfig cfg; - cfg.layerConfig.set_type(compareTypes[0]); +static void getMKLDNNConvConfig(TestConfig& cfg, const testConvDesc& pm) { + cfg.layerConfig.set_type("mkldnn_conv"); cfg.layerConfig.set_num_filters(pm.oc); cfg.layerConfig.set_size(pm.oc * pm.oh * pm.ow); - // cfg.layerConfig.set_partial_sum(1); // TODO: check it cfg.layerConfig.set_shared_biases(true); cfg.inputDefs.push_back( {INPUT_DATA, @@ -115,15 +121,14 @@ void testConvLayer(const testConvDesc& pm) { int oh = outputSize(pm.ih, fh, pm.ph, pm.sh, true); CHECK_EQ(ow, pm.ow) << "output size check failed"; CHECK_EQ(oh, pm.oh) << "output size check failed"; +} - MKLDNNTester tester; +void testConvLayer(const testConvDesc& pm) { + TestConfig dnnConfig; + getMKLDNNConvConfig(dnnConfig, pm); for (auto biasSize : {pm.oc, 0}) { - cfg.biasSize = biasSize; - TestConfig ref = cfg; - ref.layerConfig.set_type(compareTypes[1]); - for (auto bs : {pm.bs, 1}) { - tester.run(cfg, ref, bs, pm.ih, pm.iw); - } + dnnConfig.biasSize = biasSize; + RUN_MKLDNN_TEST_LAYER(dnnConfig, "exconv", pm) } } @@ -143,7 +148,7 @@ TEST(MKLDNNLayer, ConvLayer) { } struct testPoolDesc { - int bs, ch; // input channel and output channel are the same + int bs, ic; // input channel and output channel are the same int ih, iw; int oh, ow; int fh, fw; @@ -151,19 +156,18 @@ struct testPoolDesc { int sh, sw; }; -void testPoolLayer(const testPoolDesc& pm) { - const std::string compareTypes[] = {"mkldnn_pool", "pool"}; - TestConfig cfg; - cfg.layerConfig.set_type(compareTypes[0]); - cfg.layerConfig.set_size(pm.ch * pm.oh * pm.ow); +static void getMKLDNNPoolConfig(TestConfig& cfg, const testPoolDesc& pm) { + cfg.layerConfig.set_type("mkldnn_pool"); + cfg.layerConfig.set_size(pm.ic * pm.oh * pm.ow); cfg.inputDefs.push_back( {INPUT_DATA, "layer_0", - /* size of input layer= */ size_t(pm.ch * pm.ih * pm.iw), + /* size of input layer= */ size_t(pm.ic * pm.ih * pm.iw), 0}); LayerInputConfig* input = cfg.layerConfig.add_inputs(); PoolConfig* pool = input->mutable_pool_conf(); - pool->set_channels(pm.ch); + pool->set_pool_type("avg-projection"); + pool->set_channels(pm.ic); pool->set_img_size(pm.iw); pool->set_img_size_y(pm.ih); pool->set_output_x(pm.ow); @@ -179,20 +183,21 @@ void testPoolLayer(const testPoolDesc& pm) { int ow = outputSize(pm.iw, pm.fw, pm.pw, pm.sw, false); CHECK_EQ(ow, pm.ow) << "output size check failed"; CHECK_EQ(oh, pm.oh) << "output size check failed"; +} - MKLDNNTester tester; +void testPoolLayer(const testPoolDesc& pm) { + TestConfig dnnConfig; + getMKLDNNPoolConfig(dnnConfig, pm); + LayerInputConfig* input = dnnConfig.layerConfig.mutable_inputs(0); + PoolConfig* pool = input->mutable_pool_conf(); for (auto type : {"max-projection", "avg-projection"}) { pool->set_pool_type(type); - TestConfig ref = cfg; - ref.layerConfig.set_type(compareTypes[1]); - for (auto bs : {pm.bs, 1}) { - tester.run(cfg, ref, bs, pm.ih, pm.iw); - } + RUN_MKLDNN_TEST_LAYER(dnnConfig, "pool", pm) } } TEST(MKLDNNLayer, PoolLayer) { - /* bs, ch, ih, iw, oh, ow, fh, fw, ph, pw, sh, sw*/ + /* bs, ch, ih, iw, oh, ow, fh, fw, ph, pw, sh, sw */ testPoolLayer({2, 1, 4, 4, 2, 2, 3, 3, 0, 0, 2, 2}); testPoolLayer({10, 8, 16, 16, 8, 8, 2, 2, 0, 0, 2, 2}); testPoolLayer({4, 2, 5, 5, 3, 3, 3, 3, 1, 1, 2, 2}); @@ -204,44 +209,36 @@ TEST(MKLDNNLayer, PoolLayer) { } struct testActDesc { - int bs, ch; - int ih, iw; + int bs, ic, ih, iw; }; static void getAddtoConfig(TestConfig& cfg, const testActDesc& pm) { cfg.biasSize = 0; cfg.layerConfig.set_type("addto"); - cfg.layerConfig.set_size(pm.ch * pm.ih * pm.iw); - cfg.inputDefs.push_back( - {INPUT_DATA, - "layer_0", - /* size of input layer= */ size_t(pm.ch * pm.ih * pm.iw), - 0}); + size_t layerSize = pm.ih * pm.ih * pm.iw; + cfg.layerConfig.set_size(layerSize); + cfg.inputDefs.push_back({INPUT_DATA, "layer_0", layerSize, 0}); cfg.layerConfig.add_inputs(); } -void testActivation(std::string& type, const testActDesc& pm) { - const std::string compareTypes[] = {type, type.erase(0, 7)}; +void testActivation(std::string& actType, const testActDesc& pm) { + // TODO(TJ): mkldnn_softmax not implemented, paddle do not have elu activation + if (actType == "mkldnn_softmax" || actType == "mkldnn_elu") { + return; + } + const std::string compareTypes[] = {actType, actType.erase(0, 7)}; TestConfig cfg; getAddtoConfig(cfg, pm); - TestConfig ref = cfg; cfg.layerConfig.set_active_type(compareTypes[0]); ref.layerConfig.set_active_type(compareTypes[1]); - MKLDNNTester tester; - for (auto bs : {pm.bs, 1}) { - tester.run(cfg, ref, bs, pm.ih, pm.iw); - } + RUN_MKLDNN_TEST(cfg, ref, pm) } TEST(MKLDNNActivation, Activations) { auto types = MKLDNNActivation::getAllRegisteredTypes(); - // TODO(TJ): mkldnn_softmax not implemented, paddle do not have elu activation - std::set excluded{"mkldnn_softmax", "mkldnn_elu"}; for (auto type : types) { - if (excluded.count(type)) { - continue; - } + /* bs, c, h, w*/ testActivation(type, {16, 64, 32, 32}); } } From c1e3550ecc41da9df58e83e744a8785325636ca4 Mon Sep 17 00:00:00 2001 From: ranqiu Date: Wed, 20 Sep 2017 16:29:10 +0800 Subject: [PATCH 27/35] Update faq of doc --- doc/faq/index_cn.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/faq/index_cn.rst b/doc/faq/index_cn.rst index 209dac6989..00192aa69b 100644 --- a/doc/faq/index_cn.rst +++ b/doc/faq/index_cn.rst @@ -325,7 +325,7 @@ pip install python/dist/paddle*.whl && pip install ../paddle/dist/py_paddle*.whl 16. PaddlePaddle存储的参数格式是什么，如何和明文进行相互转化 --------------------------------------------------------- -PaddlePaddle保存的模型参数文件内容由16字节头信息和网络参数两部分组成。头信息中，1~4字节表示PaddlePaddle版本信息，在多数情况下，可以直接填充0；5~8字节表示每个参数占用的字节数，当保存的网络参数为float类型时为4，double类型时为8；9~16字节表示保存的参数总个数。 +PaddlePaddle保存的模型参数文件内容由16字节头信息和网络参数两部分组成。头信息中，1~4字节表示PaddlePaddle版本信息，请直接填充0；5~8字节表示每个参数占用的字节数，当保存的网络参数为float类型时为4，double类型时为8；9~16字节表示保存的参数总个数。 将PaddlePaddle保存的模型参数还原回明文时，可以使用相应数据类型的 :code:`numpy.array` 加载具体网络参数，此时可以跳过PaddlePaddle模型参数文件的头信息。若在PaddlePaddle编译时，未指定按照double精度编译，默认情况下按照float精度计算，保存的参数也是float类型。这时在使用 :code:`numpy.array` 时，一般设置 :code:`dtype=float32` 。示例如下： @@ -340,7 +340,7 @@ PaddlePaddle保存的模型参数文件内容由16字节头信息和网络参数 fmt="%.6f", delimiter=",") -将明文参数转化为PaddlePaddle可加载的模型参数时，首先构造头信息，再写入网络参数。下面将随机生成的矩阵转化为可以被PaddlePaddle加载的模型参数。 +将明文参数转化为PaddlePaddle可加载的模型参数时，首先构造头信息，再写入网络参数。下面的代码将随机生成的矩阵转化为可以被PaddlePaddle加载的模型参数。 .. code-block:: python From 4ec3a77a4366f756f0002180984c740376f6e6ef Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Wed, 20 Sep 2017 17:03:28 +0800 Subject: [PATCH 28/35] should run resetBwd before bwdAct --- paddle/gserver/layers/MKLDNNConvLayer.cpp | 3 ++- paddle/gserver/layers/MKLDNNFcLayer.cpp | 1 + paddle/gserver/layers/MKLDNNLayer.h | 12 +++++------- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/paddle/gserver/layers/MKLDNNConvLayer.cpp b/paddle/gserver/layers/MKLDNNConvLayer.cpp index 2647cb6006..88b047c89b 100644 --- a/paddle/gserver/layers/MKLDNNConvLayer.cpp +++ b/paddle/gserver/layers/MKLDNNConvLayer.cpp @@ -449,13 +449,14 @@ void MKLDNNConvLayer::resetOutGrad( cvtOutGrad_ = nullptr; if (!outputIsOnlyMKLDNN()) { const MatrixPtr& cpuOut = getOutput(CPU_DEVICE).grad; + outMat->setData(cpuOut->getData()); // same PrimitiveDesc with cpuInVal_ CHECK(cpuOutVal_); cpuOutGrad_ = MKLDNNMatrix::create(cpuOut, cpuOutVal_->getPrimitiveDesc()); if (cpuOutGrad_->getPrimitiveDesc() == out->getPrimitiveDesc()) { - outMat->setData(cpuOut->getData()); out = cpuOutGrad_; } else { + out = MKLDNNMatrix::create(nullptr, wgtPD->diff_dst_primitive_desc()); cvtOutGrad_ = MKLDNNMatrix::createReorder(cpuOutGrad_, out); CHECK(cvtOutGrad_); } diff --git a/paddle/gserver/layers/MKLDNNFcLayer.cpp b/paddle/gserver/layers/MKLDNNFcLayer.cpp index 66b358bcea..afd092666b 100644 --- a/paddle/gserver/layers/MKLDNNFcLayer.cpp +++ b/paddle/gserver/layers/MKLDNNFcLayer.cpp @@ -232,6 +232,7 @@ void MKLDNNFcLayer::resetBwdBuffers(MKLDNNMatrixPtr& in, void MKLDNNFcLayer::resetOutGrad(MKLDNNMatrixPtr& out) { // TODO(TJ): merge outgrad int device = outputIsOnlyMKLDNN() ? MKLDNN_DEVICE : CPU_DEVICE; + output_.grad->setData(getOutput(device).grad->getData()); // for MKLDNN device: // can not directly cast outputgrad to mkldnnmatrix, // since each layer can not write the inputgrad to mkldnn inputgrad. diff --git a/paddle/gserver/layers/MKLDNNLayer.h b/paddle/gserver/layers/MKLDNNLayer.h index c4e4a6874e..d8555a8331 100644 --- a/paddle/gserver/layers/MKLDNNLayer.h +++ b/paddle/gserver/layers/MKLDNNLayer.h @@ -141,18 +141,16 @@ public: } void backward(const UpdateCallback& callback) override { - /* Do derivation */ { + if (needResetBwd_) { + resetBwd(pipelineBwd_, inGrad_, wgtGrad_, biasGrad_, outGrad_); + needResetBwd_ = false; + } + { REGISTER_TIMER_INFO("BpActTimer", getName().c_str()); backwardActivation(); } - { REGISTER_TIMER_INFO("mkldnn_bwdTimer", getName().c_str()); - if (needResetBwd_) { - resetBwd(pipelineBwd_, inGrad_, wgtGrad_, biasGrad_, outGrad_); - needResetBwd_ = false; - } - stream_->submit(pipelineBwd_); } From bbd6e09c224e6c44a98af016f931545363596cfe Mon Sep 17 00:00:00 2001 From: yangyaming Date: Wed, 20 Sep 2017 17:09:07 +0800 Subject: [PATCH 29/35] Using LoDTensor for output. --- paddle/operators/modified_huber_loss_op.cc | 7 ++++--- paddle/operators/modified_huber_loss_op.h | 12 +++++++----- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/paddle/operators/modified_huber_loss_op.cc b/paddle/operators/modified_huber_loss_op.cc index a6e76c8166..6fe018f9a8 100644 --- a/paddle/operators/modified_huber_loss_op.cc +++ b/paddle/operators/modified_huber_loss_op.cc @@ -34,8 +34,8 @@ class ModifiedHuberLossOp : public framework::OperatorWithKernel { PADDLE_ENFORCE_EQ(x->dims().size(), 2, "The tensor rank of X must be 2."); PADDLE_ENFORCE_EQ(x->dims()[1], 1, "The 2nd dimension of X must be 1."); - context.Output("IntermediateVal")->Resize(x->dims()); - context.Output("Out")->Resize({x->dims()[0], 1}); + context.Output("IntermediateVal")->Resize(x->dims()); + context.Output("Out")->Resize({x->dims()[0], 1}); } }; @@ -80,7 +80,8 @@ class ModifiedHuberLossGradOp : public framework::OperatorWithKernel { auto* y = context.Input("Y"); auto* intermediate_val = context.Input("IntermediateVal"); auto* out_grad = context.Input(framework::GradVarName("Out")); - auto* x_grad = context.Output(framework::GradVarName("X")); + auto* x_grad = + context.Output(framework::GradVarName("X")); PADDLE_ENFORCE_NOT_NULL(x, "X must be initialized."); PADDLE_ENFORCE_NOT_NULL(y, "Y must be initialized."); diff --git a/paddle/operators/modified_huber_loss_op.h b/paddle/operators/modified_huber_loss_op.h index e78be06ebd..2b2aae1708 100644 --- a/paddle/operators/modified_huber_loss_op.h +++ b/paddle/operators/modified_huber_loss_op.h @@ -52,8 +52,8 @@ class ModifiedHuberLossKernel : public framework::OpKernel { void Compute(const framework::ExecutionContext& context) const override { auto* in0 = context.Input("X"); auto* in1 = context.Input("Y"); - auto* out0 = context.Output("IntermediateVal"); - auto* out1 = context.Output("Out"); + auto* out0 = context.Output("IntermediateVal"); + auto* out1 = context.Output("Out"); out0->mutable_data(context.GetPlace()); out1->mutable_data(context.GetPlace()); @@ -77,9 +77,11 @@ class ModifiedHuberLossGradCPUKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { auto* in0 = context.Input("Y"); - auto* in1 = context.Input("IntermediateVal"); - auto* in2 = context.Input(framework::GradVarName("Out")); - auto* out0 = context.Output(framework::GradVarName("X")); + auto* in1 = context.Input("IntermediateVal"); + auto* in2 = + context.Input(framework::GradVarName("Out")); + auto* out0 = + context.Output(framework::GradVarName("X")); if (out0) { const T* y_ptr = in0->data(); From 414a7a1e42fc38f39a42e260608cc0ad97868ddd Mon Sep 17 00:00:00 2001 From: Yancey Date: Wed, 20 Sep 2017 17:57:26 +0800 Subject: [PATCH 30/35] fix lod tensor doc (#4225) --- paddle/framework/lod_tensor.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/paddle/framework/lod_tensor.md b/paddle/framework/lod_tensor.md index 769b61f175..39b27d9b95 100644 --- a/paddle/framework/lod_tensor.md +++ b/paddle/framework/lod_tensor.md @@ -4,13 +4,13 @@ PaddlePaddle's RNN doesn't require that all instances have the same length. To ## Challenge of Variable-length Inputs -People usually represent a mini-batch by a Tensor. For example, a mini-batch of 32 images, each of size 32x32, is a 10x32x32 Tensor. So a transformation, T, of all images can be a matrix multiplication of the 32x32xO-dimensional tensor T and the 10x32x32 Tensor. +People usually represent a mini-batch by a Tensor. For example, a mini-batch of 10 images, each of size 32x32, is a 10x32x32 Tensor. So a transformation, T, of all images can be a matrix multiplication of the 32x32xO-dimensional tensor T and the 10x32x32 Tensor. Another example is that each mini-batch contains 32 sentences, where each word is a D-dimensional one-hot vector. If all sentences have the same length L, we can represent this mini-batch by a 32xLxD tensor. However, in most cases, sentences have variable lengths, and we will need an index data structure to record these variable lengths. ## LoD as a Solution -### Mini-Batch of variable-length sentenses +### Mini-Batch of variable-length sentences Let's imagine a mini-batch of 3 variable lengths sentences, containing 3, 1, and 2 words respectively. We can represent it by a (3+1+2)xD tensor plus some index information: @@ -51,17 +51,17 @@ The many 1's on the second level seem duplicated. For this particular case of 2 In summary, as long as that the essential elements (words or images) have the same size, we can represent mini-batches by a LoD Tensor: - The underlying tensor has size LxD1xD2x..., where D1xD2... is the size of the essential elements, and -- the first dimension size L has an additon property -- a LoD index as a nested vector: +- The first dimension size L has an additonal property -- a LoD index as a nested vector: ```c++ typedef std::vector > LoD; ``` -- The LoD index can is not necessary when there are only two levels and all elements of the second level have length 1. +- The LoD index is not necessary when there are only two levels and all elements of the second level have length 1. ## Slicing of LoD Tensor -Consider that we have a network with three levels of RNN: the top level one handles articles, the second level one handles sentences, and the basic level one handles words. This network requires that mini-batches represented by 4 level LoD Tensor, for example, +Consider that we have a network with three levels of RNN: the top level one handles articles, the second level one handles sentences, and the basic level one handles words. This network requires that mini-batches represented by 3 level LoD Tensor, for example, ``` 3 @@ -90,8 +90,9 @@ and the <1,2>-slice of above example is Let's go on slicing this slice. Its <1,1>-slice is ``` -3 -||| +1 +1 +| ``` ### The Slicing Algorithm @@ -128,7 +129,7 @@ Suppose that we want to retrieve the <1,2>-slice we will need to find out the starting position of this slice by summing over all leaf nodes in `LoD` to the left of the slice, i.e., 3 + 2 + 4 + 1 = 10. -To avoid the traversal of the LoD tree at slcing time, we can do it at the construction time -- instead of saving the lengths of the next level in the LoD tree, we can save the starting offset of the next level. For example, above LoD Tensor can be transformed into +To avoid the traversal of the LoD tree at slicing time, we can do it at the construction time -- instead of saving the lengths of the next level in the LoD tree, we can save the starting offset of the next level. For example, above LoD Tensor can be transformed into ``` 0 From d7a2290594f6ebca04b7eb165e1055a8b3fc660e Mon Sep 17 00:00:00 2001 From: hedaoyuan Date: Wed, 20 Sep 2017 19:09:41 +0800 Subject: [PATCH 31/35] Bug fix. --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4b564b4826..4921226ec1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -66,7 +66,7 @@ endif() if(ANDROID OR IOS) if(ANDROID) - if(AND ${CMAKE_SYSTEM_VERSION} VERSION_LESS "16") + if(${CMAKE_SYSTEM_VERSION} VERSION_LESS "16") message(FATAL_ERROR "Unsupport standalone toolchains with Android API level lower than 16") elseif(${CMAKE_SYSTEM_VERSION} VERSION_LESS "21") # TODO: support glog for Android api 16 ~ 19 in the future From eb26fdce4641408a89057f25a9629db31310b222 Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Wed, 20 Sep 2017 17:04:29 +0800 Subject: [PATCH 32/35] add python interface for mkldnn_relu and mkldnn_tanh --- python/paddle/trainer/config_parser.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index 7c32eb0069..0f57b81966 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -1565,6 +1565,10 @@ class LayerBase(object): self.config = g_config.model_config.layers.add() assert isinstance(self.config, LayerConfig) + use_mkldnn = bool(int(g_command_config_args.get("use_mkldnn", 0))) + mkldnn_acts = ['relu', 'tanh'] + if use_mkldnn and active_type in mkldnn_acts: + active_type = "mkldnn_" + active_type self.config.name = name self.config.type = type self.config.active_type = active_type From 9469bacef118305ab24c8f0eafe3daa19b4d4141 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Wed, 20 Sep 2017 11:12:15 -0700 Subject: [PATCH 33/35] add virtual to OpProtoAndCheckerMaker destructor --- paddle/framework/op_proto_maker.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/framework/op_proto_maker.h b/paddle/framework/op_proto_maker.h index fea15a374b..4d55a37db9 100644 --- a/paddle/framework/op_proto_maker.h +++ b/paddle/framework/op_proto_maker.h @@ -25,7 +25,7 @@ class OpProtoAndCheckerMaker { OpProtoAndCheckerMaker(OpProto* proto, OpAttrChecker* op_checker) : proto_(proto), op_checker_(op_checker) {} - ~OpProtoAndCheckerMaker() { + virtual ~OpProtoAndCheckerMaker() { PADDLE_ENFORCE(validated_, "should call Validate after build"); } From 0c98b167d0d07c8b4fb45f4be839961981ccb9e1 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Wed, 20 Sep 2017 12:12:27 -0700 Subject: [PATCH 34/35] Add program.md --- doc/design/program.md | 59 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 doc/design/program.md diff --git a/doc/design/program.md b/doc/design/program.md new file mode 100644 index 0000000000..cc1ffcbaf9 --- /dev/null +++ b/doc/design/program.md @@ -0,0 +1,59 @@ +# Design Doc: ProgramDesc + +The basic structure of a PaddlePaddle program is some nested blocks, as a C++ or Java program. + +As described in [graph.md](./graph.md), the first five lines of the following PaddlePaddle program + +```python +x = layer.data("images") +l = layer.data("label") +y = layer.fc(x) +cost = layer.mse(y, l) +optimize(cost) +train(cost, reader=mnist.train()) +``` + +generates, or compiles, a PaddelPaddle program, which is represented by the following protobuf message: + +```protobuf +message ProgramDesc { + repeated BlockDesc blocks = 1; +} + +message BlockDesc { + repeated VarDesc vars = 1; + repeated OpDesc ops = 2; +} + +message OpDesc { + AttrDesc attrs = 1; + ... +} + +message AttrDesc { + required AttrType type = 1; + + // index into ProgramDesc::blocks when type==BLOCK + optional int32 block = 2; + ... +} +``` + +When each of the first five lines runs, related Python function, e.g., `layer.fc`, calls C++ InferShape functions. This InferShape function needs to access the properties of VarDesc's accessed by the current OpDesc. These VarDesc's might not be defined in the current block, but in some ancestor blocks. This requires that we can trace the parent of a block. + +A nested block is often an attribute of an operator, most likely, an IfElseOp or a WhileOp. In above solution, all blocks are in `ProgramDesc::blocks`, this implicitly assigns a zero-based ID to each block -- the index of the block in `ProgramDesc::blocks`. So that `AttrDesc::block` could be an integer block ID. + +With this design, the InferShape function should take the following parameters: + +```c++ +void InferShape(const ProgramDesc* program, + int current_block, + int current_operator) { + ... +} +``` + +where + +- `current_block` indices into `ProgramDesc::blocks`, +- `current_operator` indices into `BlockDesc::ops`. From 34bdcac500b4d703a1309939bf44e337a2a9683e Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Wed, 20 Sep 2017 12:29:25 -0700 Subject: [PATCH 35/35] Update --- doc/design/program.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/doc/design/program.md b/doc/design/program.md index cc1ffcbaf9..fb8f86ac07 100644 --- a/doc/design/program.md +++ b/doc/design/program.md @@ -21,8 +21,9 @@ message ProgramDesc { } message BlockDesc { - repeated VarDesc vars = 1; - repeated OpDesc ops = 2; + required int32 parent = 1; + repeated VarDesc vars = 2; + repeated OpDesc ops = 3; } message OpDesc { @@ -46,9 +47,10 @@ A nested block is often an attribute of an operator, most likely, an IfElseOp or With this design, the InferShape function should take the following parameters: ```c++ -void InferShape(const ProgramDesc* program, - int current_block, - int current_operator) { +void InferShape(int current_block, + int current_operator, + ProgramDesc* program // might change VarDesc values. + ) { ... } ```