From bd66d056c8404915d21b5214e27ea1b2ffdd2892 Mon Sep 17 00:00:00 2001 From: typhoonzero Date: Fri, 8 Sep 2017 15:09:45 +0800 Subject: [PATCH 001/170] fix row buffer mem free --- paddle/math/RowBuffer.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/paddle/math/RowBuffer.h b/paddle/math/RowBuffer.h index dbb829c4e2..9ef5b89680 100644 --- a/paddle/math/RowBuffer.h +++ b/paddle/math/RowBuffer.h @@ -99,7 +99,11 @@ public: /** * @brief clear local buffer. It only affect auto-growth buffer. */ - inline void clear() { rowStore_.clear(); } + inline void clear() { + // swap an empty vector to it to free the memory. + std::vector> empty; + rowStore_.swap(empty); + } /** * @brief get current number of rows. From 345499b0feeff2a09394529fe74f5a705bbdc5e2 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Tue, 12 Sep 2017 19:17:48 -0700 Subject: [PATCH 002/170] Add Skeleton of Python API design --- doc/design/python_api.md | 104 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 doc/design/python_api.md diff --git a/doc/design/python_api.md b/doc/design/python_api.md new file mode 100644 index 0000000000..395341d37e --- /dev/null +++ b/doc/design/python_api.md @@ -0,0 +1,104 @@ +# Design Doc: Python API + + + +The top level user API in Python should be as same as API in `paddle.v2` after refactoring Paddle from a layer based framework to an operator based framework. There are many new classes in CPP in [compile time] for describing neural networks, such as `Variable`, `Operator`, `Block`. The issue about current design is how to give a proper way to wrap the C++ API to `paddle.v2` API and writing layers in Python. + + + +This implementation of Python API includes two steps. + +1. Implement the Python API using current C++ runtime concepts. +2. Replace the implementation by using compile-time concepts when they are completed. + +... + + +## Python Class about compile-time concepts + + +| Python Class | Compile-time protobuf | +| --- | --- | +| Block | BlockDesc | +| Operator | OpDesc | +| Variable | VarDesc | + + +### Block + + + +```python +class Block(objects): + def __init__(self, parent=None): + self.vars_ = map() + self.ops_ = vector() + if parent is None: + self.global_vars = map() + self.parent=None + else: + self.parent = parent + self.global_vars = None + + def create_global_vars(...): + if self.parent is not None: + return self.parent.create_global_vars(...) + else: + return self.global_vars.new() +``` + + +### Operator + + + +```python +class Operator(object): + def __init__(self, type, inputs, outputs, attrs): + # create OpDesc in Python + op_desc = ... + self.cpp_op_desc_ptr = cpp.to_cpp_op_desc(op_desc) + cpp.infer_shapes(self.cpp_op_desc_ptr, inputs, outputs) + outputs.op = self + + def type(self): + return self.cpp_op_desc_ptr.type() +``` + +### Variable + + + +```python +class Variable(object): + def __init__(self, shape, dtype="float32", name=None, block=None): + if name is None: + if prefix is not None: + name = unique_name_generator(prefix) + else: + name = unique_name_generator("unknown") + self.name = name + self.block = block + self.cpp_var_desc_ptr = ... + self.op = None + + def shape(self): + cpp_shape = self.cpp_var_desc_ptr.shape() + return [None if elem < 0 else elem for elem in cpp_shape] +``` + +### Parameter + + + + + +```python +class Parameter(Variable): + def __init__(self, trainable, initialize_attrs, optimize_attrs): + pass +``` + +## Layer Functions + + From 4b623c1e1178a2ee7dc27243cee2fbae6ea9eea4 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Tue, 12 Sep 2017 20:08:42 -0700 Subject: [PATCH 003/170] Update design --- doc/design/python_api.md | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/doc/design/python_api.md b/doc/design/python_api.md index 395341d37e..e314d1049c 100644 --- a/doc/design/python_api.md +++ b/doc/design/python_api.md @@ -1,22 +1,20 @@ # Design Doc: Python API - - The top level user API in Python should be as same as API in `paddle.v2` after refactoring Paddle from a layer based framework to an operator based framework. There are many new classes in CPP in [compile time] for describing neural networks, such as `Variable`, `Operator`, `Block`. The issue about current design is how to give a proper way to wrap the C++ API to `paddle.v2` API and writing layers in Python. - - This implementation of Python API includes two steps. 1. Implement the Python API using current C++ runtime concepts. 2. Replace the implementation by using compile-time concepts when they are completed. -... +The implementation of the first step is a temporary implementation. We should design our Python API concepts based on `compile-time` concepts. We just use `runtime` classes to implement it for now. + + +## Python Class and compile-time protobuf +As we design our Python API concepts based on `compile-time`, we try to map our Python classes to every compile-time result, i.e., the protobuf messages. They are: -## Python Class about compile-time concepts - | Python Class | Compile-time protobuf | | --- | --- | | Block | BlockDesc | From ea22f838290f2d44887b4e9da2b8b5f73669fd63 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Tue, 12 Sep 2017 20:41:21 -0700 Subject: [PATCH 004/170] Update design --- doc/design/python_api.md | 42 +++++++++++++++++++++++++++------------- 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/doc/design/python_api.md b/doc/design/python_api.md index e314d1049c..02bfcde04e 100644 --- a/doc/design/python_api.md +++ b/doc/design/python_api.md @@ -12,7 +12,7 @@ The implementation of the first step is a temporary implementation. We should de ## Python Class and compile-time protobuf -As we design our Python API concepts based on `compile-time`, we try to map our Python classes to every compile-time result, i.e., the protobuf messages. They are: +Since we design our Python API concepts based on `compile-time`, we try to map our Python classes to every compile-time result, i.e., the protobuf messages. They are: | Python Class | Compile-time protobuf | @@ -24,27 +24,43 @@ As we design our Python API concepts based on `compile-time`, we try to map our ### Block - +Block is just like programming languages `{}`, which contains many operators and variables. There are two data fields in `Block`. 1) An associate map, whose key is variable name and value is variable itself; 2) A list of operators. + +The block is hierarchical because PaddlePaddle supports RNN and IfElse. For example, RNN is like `for-loop` in programming languages. There is new `block` inside a `for-loop`. To represent hierarchies, `Block` stores the `parent Block` inside. If `parent=None`, the `Block` is the outermost block, i.e., the `global` block. + ```python class Block(objects): def __init__(self, parent=None): - self.vars_ = map() - self.ops_ = vector() - if parent is None: - self.global_vars = map() - self.parent=None - else: - self.parent = parent - self.global_vars = None + self.vars = map() + self.ops = vector() + self.parent = parent + + def create_var(self, ...): + # create variable in `self.vars` + return Variable(...) - def create_global_vars(...): + + def create_global_var(self, ...): if self.parent is not None: - return self.parent.create_global_vars(...) + return self.parent.create_global_var(...) else: - return self.global_vars.new() + return self.create_var(...) + + def create_parameter(self, ...): + return self.create_global_var(...) + + def append_operator(self, ...): + self.ops.append(...) + + def prepend_operator(self, ...): + self.ops.prepend(...) ``` +Users are able to create a global variable inside any block since they many create parameters inside a RNN or IfElseOp. All parameters should be stored in the global block, not the step block in RNN. + +Users can create local variables for outputs of operators. Users can also append and prepend an operator in current block. Prepending `random initialize` operator or `load` operator is very useful to initialize parameters before training. + ### Operator From c33a9bddfdd86a273a342c1be857423d0e331476 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Tue, 12 Sep 2017 20:52:22 -0700 Subject: [PATCH 005/170] Update doc --- doc/design/python_api.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/doc/design/python_api.md b/doc/design/python_api.md index 02bfcde04e..0a036d1613 100644 --- a/doc/design/python_api.md +++ b/doc/design/python_api.md @@ -1,6 +1,6 @@ # Design Doc: Python API -The top level user API in Python should be as same as API in `paddle.v2` after refactoring Paddle from a layer based framework to an operator based framework. There are many new classes in CPP in [compile time] for describing neural networks, such as `Variable`, `Operator`, `Block`. The issue about current design is how to give a proper way to wrap the C++ API to `paddle.v2` API and writing layers in Python. +The top level user API in Python should be as same as API in `paddle.v2` after refactoring Paddle from a layer based framework to an operator based framework. There are many new classes in C++ in [compile time] for describing neural networks, such as `Variable`, `Operator`, `Block`. The issue about current design is how to give a proper way to wrap the C++ API to `paddle.v2` API and writing layers in Python. This implementation of Python API includes two steps. @@ -64,21 +64,22 @@ Users can create local variables for outputs of operators. Users can also append ### Operator - +Operator class will take inputs, outputs and attributes of the operator into `protobuf` OpDesc and create a C++ `OpDesc` instance. The `infer_shape` perform on C++ objects. ```python class Operator(object): def __init__(self, type, inputs, outputs, attrs): # create OpDesc in Python op_desc = ... - self.cpp_op_desc_ptr = cpp.to_cpp_op_desc(op_desc) - cpp.infer_shapes(self.cpp_op_desc_ptr, inputs, outputs) - outputs.op = self + self.cpp_op_desc_ptr = core.OpDesc(op_desc) + cpp.infer_shape(self.cpp_op_desc_ptr, inputs, outputs) def type(self): return self.cpp_op_desc_ptr.type() ``` +After creating a C++ `OpDesc`, `Operator` in Python can only reads the attribute from C++ side. + ### Variable From 5f2bb1fdcb8435c39d8e74e10dc54083d7cb8f55 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Tue, 12 Sep 2017 20:55:26 -0700 Subject: [PATCH 006/170] Tab to space --- doc/design/python_api.md | 67 ++++++++++++++++++++-------------------- 1 file changed, 33 insertions(+), 34 deletions(-) diff --git a/doc/design/python_api.md b/doc/design/python_api.md index 0a036d1613..08255db6e9 100644 --- a/doc/design/python_api.md +++ b/doc/design/python_api.md @@ -31,30 +31,30 @@ The block is hierarchical because PaddlePaddle supports RNN and IfElse. For exam ```python class Block(objects): - def __init__(self, parent=None): - self.vars = map() - self.ops = vector() - self.parent = parent - - def create_var(self, ...): - # create variable in `self.vars` - return Variable(...) - - - def create_global_var(self, ...): - if self.parent is not None: - return self.parent.create_global_var(...) - else: - return self.create_var(...) - - def create_parameter(self, ...): - return self.create_global_var(...) - - def append_operator(self, ...): - self.ops.append(...) - - def prepend_operator(self, ...): - self.ops.prepend(...) + def __init__(self, parent=None): + self.vars = map() + self.ops = vector() + self.parent = parent + + def create_var(self, ...): + # create variable in `self.vars` + return Variable(...) + + + def create_global_var(self, ...): + if self.parent is not None: + return self.parent.create_global_var(...) + else: + return self.create_var(...) + + def create_parameter(self, ...): + return self.create_global_var(...) + + def append_operator(self, ...): + self.ops.append(...) + + def prepend_operator(self, ...): + self.ops.prepend(...) ``` Users are able to create a global variable inside any block since they many create parameters inside a RNN or IfElseOp. All parameters should be stored in the global block, not the step block in RNN. @@ -68,14 +68,14 @@ Operator class will take inputs, outputs and attributes of the operator into `pr ```python class Operator(object): - def __init__(self, type, inputs, outputs, attrs): - # create OpDesc in Python - op_desc = ... - self.cpp_op_desc_ptr = core.OpDesc(op_desc) - cpp.infer_shape(self.cpp_op_desc_ptr, inputs, outputs) - - def type(self): - return self.cpp_op_desc_ptr.type() + def __init__(self, type, inputs, outputs, attrs): + # create OpDesc in Python + op_desc = ... + self.cpp_op_desc_ptr = core.OpDesc(op_desc) + cpp.infer_shape(self.cpp_op_desc_ptr, inputs, outputs) + + def type(self): + return self.cpp_op_desc_ptr.type() ``` After creating a C++ `OpDesc`, `Operator` in Python can only reads the attribute from C++ side. @@ -108,8 +108,7 @@ class Variable(object): -```python -class Parameter(Variable): +```pythonVclass Parameter(Variable): def __init__(self, trainable, initialize_attrs, optimize_attrs): pass ``` From 889f5a41cacf155e89122df4f1722700ceb26464 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Tue, 12 Sep 2017 20:56:44 -0700 Subject: [PATCH 007/170] Update --- doc/design/python_api.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/design/python_api.md b/doc/design/python_api.md index 08255db6e9..05875eeb21 100644 --- a/doc/design/python_api.md +++ b/doc/design/python_api.md @@ -108,7 +108,8 @@ class Variable(object): -```pythonVclass Parameter(Variable): +```python +class Parameter(Variable): def __init__(self, trainable, initialize_attrs, optimize_attrs): pass ``` From 6bf1283cbebba1b795896c6dea4eb7f5a407e159 Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Wed, 13 Sep 2017 13:48:39 -0700 Subject: [PATCH 008/170] Add doc for `Variable` --- doc/design/python_api.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/doc/design/python_api.md b/doc/design/python_api.md index 05875eeb21..e515f8594d 100644 --- a/doc/design/python_api.md +++ b/doc/design/python_api.md @@ -82,18 +82,16 @@ After creating a C++ `OpDesc`, `Operator` in Python can only reads the attribute ### Variable - +Operators' intputs, outputs and parameters are all variables. In our design, a variable has four key attributes: its name(`name`), the block it belongs to(`block`), a pointer pointed to its C++ Protobuf object(`cpp_var_desc_ptr`), and the operator it is created by(`op`). All of these attributes are initialized in constructor, except the `op`. The `op` will keep being `None` till the variable is taken as an operator's output. ```python class Variable(object): def __init__(self, shape, dtype="float32", name=None, block=None): if name is None: - if prefix is not None: - name = unique_name_generator(prefix) - else: - name = unique_name_generator("unknown") + name = unique_name_generator() self.name = name self.block = block + # build C++ Protobuf object self.cpp_var_desc_ptr = ... self.op = None @@ -102,6 +100,10 @@ class Variable(object): return [None if elem < 0 else elem for elem in cpp_shape] ``` +Protobuf object should be created in C++ not Python because it is needed by infershape, and infershape is implementated by C++ code. The C++ Protobuf object is accessible for Python through the `cpp_var_desc_ptr` pointer. + +The user is allowed to build an variable without specifying its name. If so, it is going to be assigned with an automatically generated unique name. + ### Parameter From 216b87abd894d09177eca9b4c72c953ba0e5c451 Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Wed, 13 Sep 2017 14:41:14 -0700 Subject: [PATCH 009/170] Add doc for `Parameter` --- doc/design/python_api.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/doc/design/python_api.md b/doc/design/python_api.md index e515f8594d..4c012ddd19 100644 --- a/doc/design/python_api.md +++ b/doc/design/python_api.md @@ -82,7 +82,7 @@ After creating a C++ `OpDesc`, `Operator` in Python can only reads the attribute ### Variable -Operators' intputs, outputs and parameters are all variables. In our design, a variable has four key attributes: its name(`name`), the block it belongs to(`block`), a pointer pointed to its C++ Protobuf object(`cpp_var_desc_ptr`), and the operator it is created by(`op`). All of these attributes are initialized in constructor, except the `op`. The `op` will keep being `None` till the variable is taken as an operator's output. +Operators' inputs, outputs, and parameters are all variables. In our design, a variable has four key attributes: its name(`name`), the block it belongs to(`block`), a pointer pointed to its C++ Protobuf object(`cpp_var_desc_ptr`), and the operator it is created by(`op`). All of these attributes are initialized in the constructor, except the `op`. The `op` will keep being `None` till the variable is taken as an operator's output. ```python class Variable(object): @@ -100,15 +100,13 @@ class Variable(object): return [None if elem < 0 else elem for elem in cpp_shape] ``` -Protobuf object should be created in C++ not Python because it is needed by infershape, and infershape is implementated by C++ code. The C++ Protobuf object is accessible for Python through the `cpp_var_desc_ptr` pointer. +The Protobuf object should be created in C++ not Python because it is needed by infershape, and infershape is implemented by C++ code. The C++ Protobuf object is accessible for Python through the `cpp_var_desc_ptr`, just like how `shape()` function does. -The user is allowed to build an variable without specifying its name. If so, it is going to be assigned with an automatically generated unique name. +The user is allowed to build a variable without specifying its name. If so, it is going to be assigned with an automatically generated unique name. ### Parameter - - - +The parameter is a kind of special variable. They need to be initialized at the very beginning and updated after each batch training. So if a variable is a parameter, our compiler will add an initializer op and an optimizer op for it during the building process of computation graph. Apart from these, there is no more difference between variable and parameter. In other words, 'parameter' is only a label attached to variables, to tell the compiler these ones require additional processing. ```python class Parameter(Variable): @@ -116,6 +114,9 @@ class Parameter(Variable): pass ``` +The class `Parameter` is derived from class `Variable`. In addition to variables have, parameters are able to hold their initializing and updating information. A parameter's `self.op` will always be `None` because it can never be an operator's output. + + ## Layer Functions From 709a3481482b5b6f40e8df14949bfeb3c311738d Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Wed, 13 Sep 2017 16:07:20 -0700 Subject: [PATCH 010/170] Add doc for `Layer function` --- doc/design/python_api.md | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/doc/design/python_api.md b/doc/design/python_api.md index 4c012ddd19..2ca6dffae8 100644 --- a/doc/design/python_api.md +++ b/doc/design/python_api.md @@ -119,4 +119,38 @@ The class `Parameter` is derived from class `Variable`. In addition to variables ## Layer Functions - +A layer is a Python function. When it is invoked, it creates a serise of operators and variables then inserts them into the block. It is something like the macro in C++. It is called 'Layer' because the combination of added operators acts just like what a neural network layer does. + +Here are examples about how to write a date layer and FC layer: + +### Data Layer + +```python +def data_layer(name, type, block=None): + if block is None: + block = g_block + # type = dense_vector(size=10) / integer_value(range=10) + return block.create_global_var( + name=name, + shape=[None] + type.dims(), + dtype=type.dtype) + +``` + +Before building new variables, we need to specify which block to use. If we don't, the default one `g_block` will be used. In the above `data_layer` code, a variable is created and be inserted into the root block to make it global. This varibale is going to be used as input data of the whole network. + +### FC Layer + +```python +def fc_layer(input, size, block=None, ...): + if block is None: + block = g_block + w = block.create_parameter(...) + b = block.create_parameter(...) + out = stack.create_var() + op = block.append_operator(Operator("FC", X=input, W=w, b=b, Out=out)) + out.op = op + return out +``` + +In the `fc_layer` code, we create two parameters(`w` and `b`), one variable(`out`) and one operator(`FC operator`), then insert all of them into specify block. From 6b222d550bbeeac6d1278ee6b7a4145f7ba8d6d3 Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Wed, 13 Sep 2017 16:10:24 -0700 Subject: [PATCH 011/170] Fix typo --- doc/design/python_api.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/design/python_api.md b/doc/design/python_api.md index 2ca6dffae8..f0a300bed0 100644 --- a/doc/design/python_api.md +++ b/doc/design/python_api.md @@ -119,9 +119,9 @@ The class `Parameter` is derived from class `Variable`. In addition to variables ## Layer Functions -A layer is a Python function. When it is invoked, it creates a serise of operators and variables then inserts them into the block. It is something like the macro in C++. It is called 'Layer' because the combination of added operators acts just like what a neural network layer does. +A layer is a Python function. When it is invoked, it creates a series of operators and variables then inserts them into the block. It is something like the macro in C++. It is called 'Layer' because the combination of added operators acts just like what a neural network layer does. -Here are examples about how to write a date layer and FC layer: +Here are examples of how to write a data layer and FC layer: ### Data Layer @@ -137,7 +137,7 @@ def data_layer(name, type, block=None): ``` -Before building new variables, we need to specify which block to use. If we don't, the default one `g_block` will be used. In the above `data_layer` code, a variable is created and be inserted into the root block to make it global. This varibale is going to be used as input data of the whole network. +Before building new variables, we need to specify which block to use. If we don't, the default one `g_block` will be used. In the above `data_layer` code, a variable is created and be inserted into the root block to make it global. This variable is going to be used as input data of the whole network. ### FC Layer @@ -153,4 +153,4 @@ def fc_layer(input, size, block=None, ...): return out ``` -In the `fc_layer` code, we create two parameters(`w` and `b`), one variable(`out`) and one operator(`FC operator`), then insert all of them into specify block. +In the `fc_layer` code, we create two parameters(`w` and `b`), one variable(`out`) and one operator(`FC operator`), then insert all of them into the specified block. From 9e7c0b5ef7a1c6854be4c6a4db130a2d717d9d89 Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Wed, 20 Sep 2017 16:25:41 +0800 Subject: [PATCH 012/170] Add pooling2d(max, ave) and pooling3d(max, ave) functor --- paddle/operators/math/pooling.cc | 317 +++++++++++++++++++++++ paddle/operators/math/pooling.cu | 423 +++++++++++++++++++++++++++++++ paddle/operators/math/pooling.h | 99 ++++++++ 3 files changed, 839 insertions(+) create mode 100644 paddle/operators/math/pooling.cc create mode 100644 paddle/operators/math/pooling.cu create mode 100644 paddle/operators/math/pooling.h diff --git a/paddle/operators/math/pooling.cc b/paddle/operators/math/pooling.cc new file mode 100644 index 0000000000..6146e710fc --- /dev/null +++ b/paddle/operators/math/pooling.cc @@ -0,0 +1,317 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/operators/math/pooling.h" + +namespace paddle { +namespace operators { +namespace math { + +template +class Pool2dForwardFunctor { + public: + void operator()(const framework::Tensor& input, framework::Tensor& output, + std::vector& ksize, std::vector& strides, + std::vector& paddings, PoolProcess pool_process, + platform::DeviceContext* context) { + const int batch_size = input.dims()[0]; + const int input_height = input.dims()[2]; + const int input_width = input.dims()[3]; + const int output_channels = output.dims()[1]; + const int output_height = output.dims()[2]; + const int output_width = output.dims()[3]; + const int ksize_height = ksize[0]; + const int ksize_width = ksize[1]; + const int stride_height = strides[0]; + const int stride_width = strides[1]; + const int padding_height = paddings[0]; + const int padding_width = paddings[1]; + + const int input_stride = input_height * input_width; + const int output_stride = output_height * output_width; + + const T* input_data = input.data(); + T* output_data = output.mutable_data(context->GetPlace()); + + for (int i = 0; i < batch_size; i++) { + for (int c = 0; c < output_channels; ++c) { + for (int ph = 0; ph < output_height; ++ph) { + int hstart = ph * stride_height - padding_height; + int hend = std::min(hstart + ksize_height, input_height); + hstart = std::max(hstart, 0); + for (int pw = 0; pw < output_width; ++pw) { + int wstart = pw * stride_width - padding_width; + int wend = std::min(wstart + ksize_width, input_width); + wstart = std::max(wstart, 0); + T ele = pool_process.initial(); + for (int h = hstart; h < hend; ++h) { + for (int w = wstart; w < wend; ++w) { + pool_process.process(ele, input_data[h * input_width + w]); + } + } + int pool_size = (hend - hstart) * (wend - wstart); + pool_process.finalize(ele, (static_cast(pool_size))); + output_data[ph * output_width + pw] = ele; + } + } + input_data += input_stride; + output_data += output_stride; + } + } + } +}; + +template +class Pool2dBackwardFunctor { + public: + void operator()(const framework::Tensor& input, framework::Tensor& input_grad, + const framework::Tensor& output, + const framework::Tensor& output_grad, std::vector& ksize, + std::vector& strides, std::vector& paddings, + PoolProcess pool_process, platform::DeviceContext* context) { + const int batch_size = input.dims()[0]; + const int input_height = input.dims()[2]; + const int input_width = input.dims()[3]; + const int output_channels = output.dims()[1]; + const int output_height = output.dims()[2]; + const int output_width = output.dims()[3]; + const int ksize_height = ksize[0]; + const int ksize_width = ksize[1]; + const int stride_height = strides[0]; + const int stride_width = strides[1]; + const int padding_height = paddings[0]; + const int padding_width = paddings[1]; + const int input_stride = input_height * input_width; + const int output_stride = output_height * output_width; + + const T* input_data = input.data(); + const T* output_data = output.data(); + const T* output_grad_data = output_grad.data(); + T* input_grad_data = input_grad.mutable_data(context->GetPlace()); + + for (int i = 0; i < batch_size; i++) { + for (int c = 0; c < output_channels; ++c) { + for (int ph = 0; ph < output_height; ++ph) { + int hstart = ph * stride_height - padding_height; + int hend = std::min(hstart + ksize_height, input_height); + hstart = std::max(hstart, 0); + for (int pw = 0; pw < output_width; ++pw) { + int wstart = pw * stride_width - padding_width; + int wend = std::min(wstart + ksize_width, input_width); + wstart = std::max(wstart, 0); + int pool_size = (hend - hstart) * (wend - wstart); + for (int h = hstart; h < hend; ++h) { + for (int w = wstart; w < wend; ++w) { + pool_process.gradProcess( + input_data[h * input_width + w], + output_data[ph * output_width + pw], + output_grad_data[ph * output_width + pw], + input_grad_data[h * input_width + w], + static_cast(pool_size)); + } + } + } + } + input_data += input_stride; + output_data += output_stride; + input_grad_data += input_stride; + output_grad_data += output_stride; + } + } + } +}; + +template class Pool2dForwardFunctor< + platform::CPUPlace, paddle::operators::math::pool::maxPool, float>; +template class Pool2dForwardFunctor< + platform::CPUPlace, paddle::operators::math::pool::avePool, float>; +template class Pool2dBackwardFunctor< + platform::CPUPlace, paddle::operators::math::pool::maxPool, float>; +template class Pool2dBackwardFunctor< + platform::CPUPlace, paddle::operators::math::pool::avePool, float>; +template class Pool2dForwardFunctor< + platform::CPUPlace, paddle::operators::math::pool::maxPool, double>; +template class Pool2dForwardFunctor< + platform::CPUPlace, paddle::operators::math::pool::avePool, double>; +template class Pool2dBackwardFunctor< + platform::CPUPlace, paddle::operators::math::pool::maxPool, double>; +template class Pool2dBackwardFunctor< + platform::CPUPlace, paddle::operators::math::pool::avePool, double>; + +template +class Pool3dForwardFunctor { + public: + void operator()(const framework::Tensor& input, framework::Tensor& output, + std::vector& ksize, std::vector& strides, + std::vector& paddings, PoolProcess pool_process, + platform::DeviceContext* context) { + const int batch_size = input.dims()[0]; + const int input_depth = input.dims()[2]; + const int input_height = input.dims()[3]; + const int input_width = input.dims()[4]; + const int output_channels = output.dims()[1]; + const int output_depth = output.dims()[2]; + const int output_height = output.dims()[3]; + const int output_width = output.dims()[4]; + const int ksize_depth = ksize[0]; + const int ksize_height = ksize[1]; + const int ksize_width = ksize[2]; + const int stride_depth = strides[0]; + const int stride_height = strides[1]; + const int stride_width = strides[2]; + const int padding_depth = paddings[0]; + const int padding_height = paddings[1]; + const int padding_width = paddings[2]; + + const int input_stride = input_depth * input_height * input_width; + const int output_stride = output_depth * output_height * output_width; + + const T* input_data = input.data(); + T* output_data = output.mutable_data(context->GetPlace()); + + for (int i = 0; i < batch_size; i++) { + for (int c = 0; c < output_channels; ++c) { + for (int pd = 0; pd < output_depth; ++pd) { + int dstart = pd * stride_depth - padding_depth; + int dend = std::min(dstart + ksize_depth, input_depth); + dstart = std::max(dstart, 0); + for (int ph = 0; ph < output_height; ++ph) { + int hstart = ph * stride_height - padding_height; + int hend = std::min(hstart + ksize_height, input_height); + hstart = std::max(hstart, 0); + for (int pw = 0; pw < output_width; ++pw) { + int wstart = pw * stride_width - padding_width; + int wend = std::min(wstart + ksize_width, input_width); + wstart = std::max(wstart, 0); + int output_idx = (pd * output_height + ph) * output_width + pw; + T ele = pool_process.initial(); + for (int d = dstart; d < dend; ++d) { + for (int h = hstart; h < hend; ++h) { + for (int w = wstart; w < wend; ++w) { + pool_process.process( + ele, + input_data[(d * input_height + h) * input_width + w]); + } + } + } + int pool_size = + (dend - dstart) * (hend - hstart) * (wend - wstart); + pool_process.finalize(ele, static_cast(pool_size)); + output_data[output_idx] = ele; + } + } + } + input_data += input_stride; + output_data += output_stride; + } + } + } +}; + +template +class Pool3dBackwardFunctor { + public: + void operator()(const framework::Tensor& input, framework::Tensor& input_grad, + const framework::Tensor& output, + const framework::Tensor& output_grad, std::vector& ksize, + std::vector& strides, std::vector& paddings, + PoolProcess pool_process, platform::DeviceContext* context) { + const int batch_size = input.dims()[0]; + const int input_depth = input.dims()[2]; + const int input_height = input.dims()[3]; + const int input_width = input.dims()[4]; + const int output_channels = output.dims()[1]; + const int output_depth = output.dims()[2]; + const int output_height = output.dims()[3]; + const int output_width = output.dims()[4]; + const int ksize_depth = ksize[0]; + const int ksize_height = ksize[1]; + const int ksize_width = ksize[2]; + const int stride_depth = strides[0]; + const int stride_height = strides[1]; + const int stride_width = strides[2]; + const int padding_depth = paddings[0]; + const int padding_height = paddings[1]; + const int padding_width = paddings[2]; + + const int input_stride = input_depth * input_height * input_width; + const int output_stride = output_depth * output_height * output_width; + + const T* input_data = input.data(); + const T* output_data = output.data(); + const T* output_grad_data = output_grad.data(); + T* input_grad_data = input_grad.mutable_data(context->GetPlace()); + + for (int i = 0; i < batch_size; i++) { + for (int c = 0; c < output_channels; ++c) { + for (int pd = 0; pd < output_depth; ++pd) { + int dstart = pd * stride_depth - padding_depth; + int dend = std::min(dstart + ksize_depth, input_depth); + dstart = std::max(dstart, 0); + for (int ph = 0; ph < output_height; ++ph) { + int hstart = ph * stride_height - padding_height; + int hend = std::min(hstart + ksize_height, input_height); + hstart = std::max(hstart, 0); + + for (int pw = 0; pw < output_width; ++pw) { + int wstart = pw * stride_width - padding_width; + int wend = std::min(wstart + ksize_width, input_width); + wstart = std::max(wstart, 0); + + int pool_size = + (dend - dstart) * (hend - hstart) * (wend - wstart); + for (int d = dstart; d < dend; ++d) { + for (int h = hstart; h < hend; ++h) { + for (int w = wstart; w < wend; ++w) { + int input_idx = (d * input_height + h) * input_width + w; + int output_idx = + (pd * output_height + ph) * output_width + pw; + pool_process.gradProcess( + input_data[input_idx], output_data[output_idx], + output_grad_data[output_idx], + input_grad_data[input_idx], static_cast(pool_size)); + } + } + } + } + } + input_data += input_stride; + output_data += output_stride; + input_grad_data += input_stride; + output_grad_data += output_stride; + } + } + } + } +}; + +template class Pool3dForwardFunctor< + platform::CPUPlace, paddle::operators::math::pool::maxPool, float>; +template class Pool3dForwardFunctor< + platform::CPUPlace, paddle::operators::math::pool::avePool, float>; +template class Pool3dBackwardFunctor< + platform::CPUPlace, paddle::operators::math::pool::maxPool, float>; +template class Pool3dBackwardFunctor< + platform::CPUPlace, paddle::operators::math::pool::avePool, float>; +template class Pool3dForwardFunctor< + platform::CPUPlace, paddle::operators::math::pool::maxPool, double>; +template class Pool3dForwardFunctor< + platform::CPUPlace, paddle::operators::math::pool::avePool, double>; +template class Pool3dBackwardFunctor< + platform::CPUPlace, paddle::operators::math::pool::maxPool, double>; +template class Pool3dBackwardFunctor< + platform::CPUPlace, paddle::operators::math::pool::avePool, double>; +} // namespace math +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/math/pooling.cu b/paddle/operators/math/pooling.cu new file mode 100644 index 0000000000..87bfb43c47 --- /dev/null +++ b/paddle/operators/math/pooling.cu @@ -0,0 +1,423 @@ +/* Copyright (c) 2016 paddlepaddle Authors. All Rights +Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/operators/math/pooling.h" + +namespace paddle { +namespace operators { +namespace math { + +template +__global__ void KernelPool2dForward( + const int nthreads, const T* input_data, T* output_data, const int channels, + const int input_height, const int input_width, const int output_height, + const int output_width, const int ksize_height, const int ksize_width, + const int stride_height, const int stride_width, const int padding_height, + const int padding_width, PoolProcess pool_process) { + int index = blockIdx.x * blockDim.x + threadIdx.x; + if (index < nthreads) { + int pw = index % output_width; + int ph = (index / output_width) % output_height; + int c = (index / output_width / output_height) % channels; + int batch_idx = index / output_width / output_height / channels; + + int hstart = ph * stride_height - padding_height; + int hend = min(hstart + ksize_height, input_height); + hstart = max(hstart, 0); + + int wstart = pw * stride_width - padding_width; + int wend = min(wstart + ksize_width, input_width); + wstart = max(wstart, 0); + + input_data += (batch_idx * channels + c) * input_height * input_width; + T ele = pool_process.initial(); + for (int h = hstart; h < hend; ++h) { + for (int w = wstart; w < wend; ++w) { + pool_process.process(ele, input_data[h * input_width + w]); + } + } + int pool_size = (hend - hstart) * (wend - wstart); + pool_process.finalize(ele, (static_cast(pool_size))); + output_data[index] = ele; + } +} + +template +__global__ void KernelPool2dBackward( + const int nthreads, const T* input_data, const T* output_data, + const T* output_grad, T* input_grad, const int channels, + const int input_height, const int input_width, const int output_height, + const int output_width, const int ksize_height, const int ksize_width, + const int stride_height, const int stride_width, const int padding_height, + const int padding_width, PoolProcess pool_process) { + int index = blockIdx.x * blockDim.x + threadIdx.x; + if (index < nthreads) { + int offsetW = index % input_width + padding_width; + int offsetH = (index / input_width) % input_height + padding_height; + int offsetC = (index / input_width / input_height) % channels; + int batch_idx = index / input_width / input_height / channels; + + int phstart = (offsetH < ksize_height) + ? 0 + : (offsetH - ksize_height) / stride_height + 1; + int pwstart = (offsetW < ksize_width) + ? 0 + : (offsetW - ksize_width) / stride_width + 1; + int phend = min(offsetH / stride_height + 1, output_height); + int pwend = min(offsetW / stride_width + 1, output_width); + T gradient = 0; + T input = input_data[index]; + int output_idx = + (batch_idx * channels + offsetC) * output_height * output_width; + output_data += output_idx; + output_grad += output_idx; + for (int ph = phstart; ph < phend; ++ph) { + for (int pw = pwstart; pw < pwend; ++pw) { + int hstart = ph * stride_height - padding_height; + int wstart = pw * stride_width - padding_width; + int hend = min(hstart + ksize_height, input_height); + int wend = min(wstart + ksize_width, input_width); + hstart = max(hstart, 0); + wstart = max(wstart, 0); + int pool_size = (hend - hstart) * (wend - wstart); + int output_sub_idx = ph * output_width + pw; + pool_process.gradProcess(input, output_data[output_sub_idx], + output_grad[output_sub_idx], gradient, + static_cast(pool_size)); + } + } + input_grad[index] = gradient; + } +} + +template +class Pool2dForwardFunctor { + public: + void operator()(const framework::Tensor& input, framework::Tensor& output, + std::vector& ksize, std::vector& strides, + std::vector& paddings, PoolProcess pool_process, + platform::DeviceContext* context) { + const int batch_size = input.dims()[0]; + const int input_channels = input.dims()[1]; + const int input_height = input.dims()[2]; + const int input_width = input.dims()[3]; + const int output_channels = output.dims()[1]; + const int output_height = output.dims()[2]; + const int output_width = output.dims()[3]; + const int ksize_height = ksize[0]; + const int ksize_width = ksize[1]; + const int stride_height = strides[0]; + const int stride_width = strides[1]; + const int padding_height = paddings[0]; + const int padding_width = paddings[1]; + + const T* input_data = input.data(); + T* output_data = output.mutable_data(context->GetPlace()); + + int nthreads = batch_size * output_channels * output_height * output_width; + int blocks = (nthreads + 1024 - 1) / 1024; + dim3 threads(1024, 1); + dim3 grid(blocks, 1); + + KernelPool2dForward<<>>( + nthreads, input_data, output_data, input_channels, input_height, + input_width, output_height, output_width, ksize_height, ksize_width, + stride_height, stride_width, padding_height, padding_width, + pool_process); + + // CHECK_SYNC("Pool2dForwardKernel failed"); + } +}; + +template +class Pool2dBackwardFunctor { + public: + void operator()(const framework::Tensor& input, framework::Tensor& input_grad, + const framework::Tensor& output, + const framework::Tensor& output_grad, std::vector& ksize, + std::vector& strides, std::vector& paddings, + PoolProcess pool_process, platform::DeviceContext* context) { + const int batch_size = input.dims()[0]; + const int input_channels = input.dims()[1]; + const int input_height = input.dims()[2]; + const int input_width = input.dims()[3]; + const int output_height = output.dims()[2]; + const int output_width = output.dims()[3]; + const int ksize_height = ksize[0]; + const int ksize_width = ksize[1]; + const int stride_height = strides[0]; + const int stride_width = strides[1]; + const int padding_height = paddings[0]; + const int padding_width = paddings[1]; + + const T* input_data = input.data(); + const T* output_data = output.data(); + const T* output_grad_data = output_grad.data(); + T* input_grad_data = input_grad.mutable_data(context->GetPlace()); + + int nthreads = batch_size * input_channels * input_height * input_width; + int blocks = (nthreads + 1024 - 1) / 1024; + dim3 threads(1024, 1); + dim3 grid(blocks, 1); + + KernelPool2dBackward<<>>( + nthreads, input_data, output_data, output_grad_data, input_grad_data, + input_channels, input_height, input_width, output_height, output_width, + ksize_height, ksize_width, stride_height, stride_width, padding_height, + padding_width, pool_process); + + // CHECK_SYNC("KernelPool2dBackward failed"); + } +}; + +template class Pool2dForwardFunctor< + platform::GPUPlace, paddle::operators::math::pool::maxPool, float>; +template class Pool2dForwardFunctor< + platform::GPUPlace, paddle::operators::math::pool::avePool, float>; +template class Pool2dBackwardFunctor< + platform::GPUPlace, paddle::operators::math::pool::maxPool, float>; +template class Pool2dBackwardFunctor< + platform::GPUPlace, paddle::operators::math::pool::avePool, float>; +template class Pool2dForwardFunctor< + platform::GPUPlace, paddle::operators::math::pool::maxPool, double>; +template class Pool2dForwardFunctor< + platform::GPUPlace, paddle::operators::math::pool::avePool, double>; +template class Pool2dBackwardFunctor< + platform::GPUPlace, paddle::operators::math::pool::maxPool, double>; +template class Pool2dBackwardFunctor< + platform::GPUPlace, paddle::operators::math::pool::avePool, double>; + +template +__global__ void KernelPool3DForward( + const int nthreads, const T* input_data, T* output_data, const int channels, + const int input_depth, const int input_height, const int input_width, + const int output_depth, const int output_height, const int output_width, + const int ksize_depth, const int ksize_height, const int ksize_width, + const int stride_depth, const int stride_height, const int stride_width, + const int padding_depth, const int padding_height, const int padding_width, + PoolProcess pool_process) { + for (int index = blockIdx.x * blockDim.x + threadIdx.x; index < (nthreads); + index += blockDim.x * gridDim.x) { + int pw = index % output_width; + int ph = (index / output_width) % output_height; + int pd = (index / output_width / output_height) % output_depth; + int c = (index / output_width / output_height / output_depth) % channels; + int batch_idx = + index / output_width / output_height / output_depth / channels; + int dstart = pd * stride_depth - padding_depth; + int hstart = ph * stride_height - padding_height; + int wstart = pw * stride_width - padding_width; + int dend = min(dstart + ksize_depth, input_depth); + int hend = min(hstart + ksize_height, input_height); + int wend = min(wstart + ksize_width, input_width); + dstart = max(dstart, 0); + hstart = max(hstart, 0); + wstart = max(wstart, 0); + T ele = pool_process.initial(); + input_data += + (batch_idx * channels + c) * input_depth * input_height * input_width; + for (int d = dstart; d < dend; ++d) { + for (int h = hstart; h < hend; ++h) { + for (int w = wstart; w < wend; ++w) { + pool_process.process( + ele, input_data[(d * input_height + h) * input_width + w]); + } + } + } + int pool_size = (dend - dstart) * (hend - hstart) * (wend - wstart); + pool_process.finalize(ele, static_cast(pool_size)); + + output_data[index] = ele; + } +} + +template +__global__ void KernelPool3DBackward( + const int nthreads, const T* input_data, const T* output_data, + const T* output_grad, T* input_grad, const int channels, + const int input_depth, const int input_height, const int input_width, + const int output_depth, const int output_height, const int output_width, + const int ksize_depth, const int ksize_height, const int ksize_width, + const int stride_depth, const int stride_height, const int stride_width, + const int padding_depth, const int padding_height, const int padding_width, + PoolProcess pool_process) { + for (int index = blockIdx.x * blockDim.x + threadIdx.x; index < (nthreads); + index += blockDim.x * gridDim.x) { + int offsetW = index % input_width + padding_width; + int offsetH = (index / input_width) % input_height + padding_height; + int offsetD = + (index / input_width / input_height) % input_depth + padding_depth; + int offsetC = (index / input_width / input_height / input_depth) % channels; + int batch_idx = index / input_width / input_height / input_depth / channels; + + int pdstart = (offsetD < ksize_depth) + ? 0 + : (offsetD + ksize_depth) / stride_depth + 1; + int phstart = (offsetH < ksize_height) + ? 0 + : (offsetH - ksize_height) / stride_height + 1; + int pwstart = (offsetW < ksize_width) + ? 0 + : (offsetW - ksize_width) / stride_width + 1; + int pdend = min((offsetD) / stride_depth + 1, output_depth); + int phend = min((offsetH) / stride_height + 1, output_height); + int pwend = min((offsetW) / stride_width + 1, output_width); + + T gradient = 0; + T input = input_data[index]; + int output_idx = (batch_idx * channels + offsetC) * output_depth * + output_height * output_width; + output_data += output_idx; + output_grad += output_idx; + + for (int pd = pdstart; pd < pdend; ++pd) { + for (int ph = phstart; ph < phend; ++ph) { + for (int pw = pwstart; pw < pwend; ++pw) { + // figure out the pooling size + int dstart = pd * stride_depth - padding_depth; + int hstart = ph * stride_height - padding_height; + int wstart = pw * stride_width - padding_width; + int dend = min(dstart + ksize_depth, input_depth); + int hend = min(hstart + ksize_height, input_height); + int wend = min(wstart + ksize_width, input_width); + dstart = max(dstart, 0); + hstart = max(hstart, 0); + wstart = max(wstart, 0); + int pool_size = (dend - dstart) * (hend - hstart) * (wend - wstart); + int output_sub_idx = ph * output_width + pw; + pool_process.gradProcess(input, output_data[output_sub_idx], + output_grad[output_sub_idx], gradient, + static_cast(pool_size)); + } + } + } + input_grad[index] = gradient; + } +} + +template +class Pool3dForwardFunctor { + public: + void operator()(const framework::Tensor& input, framework::Tensor& output, + std::vector& ksize, std::vector& strides, + std::vector& paddings, PoolProcess pool_process, + platform::DeviceContext* context) { + const int batch_size = input.dims()[0]; + const int input_channels = input.dims()[1]; + const int input_depth = input.dims()[2]; + const int input_height = input.dims()[3]; + const int input_width = input.dims()[4]; + const int output_channels = output.dims()[1]; + const int output_depth = output.dims()[2]; + const int output_height = output.dims()[3]; + const int output_width = output.dims()[4]; + const int ksize_depth = ksize[0]; + const int ksize_height = ksize[1]; + const int ksize_width = ksize[2]; + const int stride_depth = strides[0]; + const int stride_height = strides[1]; + const int stride_width = strides[2]; + const int padding_depth = paddings[0]; + const int padding_height = paddings[1]; + const int padding_width = paddings[2]; + + const T* input_data = input.data(); + T* output_data = output.mutable_data(context->GetPlace()); + + int nthreads = batch_size * output_channels * output_depth * output_height * + output_width; + int blocks = (nthreads + 1024 - 1) / 1024; + dim3 threads(1024, 1); + dim3 grid(blocks, 1); + + KernelPool3DForward<<>>( + nthreads, input_data, output_data, input_channels, input_depth, + input_height, input_width, output_depth, output_height, output_width, + ksize_depth, ksize_height, ksize_width, stride_depth, stride_height, + stride_width, padding_depth, padding_height, padding_width, + pool_process); + + // CHECK_SYNC("Pool2dForwardKernel failed"); + } +}; + +template +class Pool3dBackwardFunctor { + public: + void operator()(const framework::Tensor& input, framework::Tensor& input_grad, + const framework::Tensor& output, + const framework::Tensor& output_grad, std::vector& ksize, + std::vector& strides, std::vector& paddings, + PoolProcess pool_process, platform::DeviceContext* context) { + const int batch_size = input.dims()[0]; + const int input_channels = input.dims()[1]; + const int input_depth = input.dims()[2]; + const int input_height = input.dims()[3]; + const int input_width = input.dims()[4]; + const int output_channels = output.dims()[1]; + const int output_depth = output.dims()[2]; + const int output_height = output.dims()[3]; + const int output_width = output.dims()[4]; + const int ksize_depth = ksize[0]; + const int ksize_height = ksize[1]; + const int ksize_width = ksize[2]; + const int stride_depth = strides[0]; + const int stride_height = strides[1]; + const int stride_width = strides[2]; + const int padding_depth = paddings[0]; + const int padding_height = paddings[1]; + const int padding_width = paddings[2]; + + const T* input_data = input.data(); + const T* output_data = output.data(); + const T* output_grad_data = output_grad.data(); + T* input_grad_data = input_grad.mutable_data(context->GetPlace()); + + int nthreads = batch_size * input_channels * input_height * input_width; + int blocks = (nthreads + 1024 - 1) / 1024; + dim3 threads(1024, 1); + dim3 grid(blocks, 1); + + KernelPool3DBackward<<>>( + nthreads, input_data, output_data, output_grad_data, input_grad_data, + input_channels, input_depth, input_height, input_width, output_depth, + output_height, output_width, ksize_depth, ksize_height, ksize_width, + stride_depth, stride_height, stride_width, padding_depth, + padding_height, padding_width, pool_process); + + // CHECK_SYNC("KernelPool2dBackward failed"); + } +}; + +template class Pool3dForwardFunctor< + platform::GPUPlace, paddle::operators::math::pool::maxPool, float>; +template class Pool3dForwardFunctor< + platform::GPUPlace, paddle::operators::math::pool::avePool, float>; +template class Pool3dBackwardFunctor< + platform::GPUPlace, paddle::operators::math::pool::maxPool, float>; +template class Pool3dBackwardFunctor< + platform::GPUPlace, paddle::operators::math::pool::avePool, float>; +template class Pool3dForwardFunctor< + platform::GPUPlace, paddle::operators::math::pool::maxPool, double>; +template class Pool3dForwardFunctor< + platform::GPUPlace, paddle::operators::math::pool::avePool, double>; +template class Pool3dBackwardFunctor< + platform::GPUPlace, paddle::operators::math::pool::maxPool, double>; +template class Pool3dBackwardFunctor< + platform::GPUPlace, paddle::operators::math::pool::avePool, double>; + +} // namespace math +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/math/pooling.h b/paddle/operators/math/pooling.h new file mode 100644 index 0000000000..69c86c6187 --- /dev/null +++ b/paddle/operators/math/pooling.h @@ -0,0 +1,99 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once +#include "paddle/framework/eigen.h" +#include "paddle/framework/tensor.h" +#include "paddle/platform/device_context.h" + +namespace paddle { +namespace operators { +namespace math { + +////////////////////// +#ifdef __NVCC__ +#define HL_DEVICE __device__ +#else +#define HL_DEVICE +#endif +#define FLT_MAX __FLT_MAX__ +///////////////////// + +namespace pool { +template +class maxPool { + public: + HL_DEVICE inline T initial() { return -(T)(FLT_MAX); } + HL_DEVICE inline void process(T& y, const T& x) { y = y > x ? y : x; } + HL_DEVICE inline void finalize(T& y, const T& poo_size) {} + HL_DEVICE inline void gradProcess(const T& x, const T& y, const T& dy, T& dx, + T scale) { + dx += dy * (x == y); + } +}; + +template +class avePool { + public: + HL_DEVICE inline T initial() { return 0; } + HL_DEVICE inline void process(T& y, const T& x) { y += x; } + HL_DEVICE inline void finalize(T& y, const T& poo_size) { y /= poo_size; } + HL_DEVICE inline void gradProcess(const T& x, const T& y, const T& dy, T& dx, + T scale) { + dx += (scale * dy); + } +}; +} // namespace pool + +template +class Pool2dForwardFunctor { + public: + void operator()(const framework::Tensor& input, framework::Tensor& output, + std::vector& ksize, std::vector& strides, + std::vector& paddings, PoolProcess pool_process, + platform::DeviceContext* context); +}; + +template +class Pool2dBackwardFunctor { + public: + void operator()(const framework::Tensor& input, framework::Tensor& input_grad, + const framework::Tensor& output, + const framework::Tensor& output_grad, std::vector& ksize, + std::vector& strides, std::vector& paddings, + PoolProcess pool_process, platform::DeviceContext* context); +}; + +template +class Pool3dForwardFunctor { + public: + void operator()(const framework::Tensor& input, framework::Tensor& output, + std::vector& ksize, std::vector& strides, + std::vector& paddings, PoolProcess pool_process, + platform::DeviceContext* context); +}; + +template +class Pool3dBackwardFunctor { + public: + void operator()(const framework::Tensor& input, framework::Tensor& input_grad, + const framework::Tensor& output, + const framework::Tensor& output_grad, std::vector& ksize, + std::vector& strides, std::vector& paddings, + PoolProcess pool_process, platform::DeviceContext* context); +}; + +} // namespace math +} // namespace operators +} // namespace paddle From 3c63b0bc4bf90befcd08fe7620ed0329f79e5fdd Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Wed, 20 Sep 2017 10:25:54 -0700 Subject: [PATCH 013/170] Change typo --- doc/design/python_api.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/design/python_api.md b/doc/design/python_api.md index f0a300bed0..74d4d3239a 100644 --- a/doc/design/python_api.md +++ b/doc/design/python_api.md @@ -1,6 +1,6 @@ # Design Doc: Python API -The top level user API in Python should be as same as API in `paddle.v2` after refactoring Paddle from a layer based framework to an operator based framework. There are many new classes in C++ in [compile time] for describing neural networks, such as `Variable`, `Operator`, `Block`. The issue about current design is how to give a proper way to wrap the C++ API to `paddle.v2` API and writing layers in Python. +The top level user API in Python should be as same as API in `paddle.v2` after refactoring Paddle from a layer based framework to an operator based framework. There are many new classes in C++ in [compile time] for describing neural networks, such as `Variable`, `Operator`, `Block`. The issue about current design is how to give a proper way to wrap the C++ API to `paddle.v2` API and write layers in Python. This implementation of Python API includes two steps. @@ -57,7 +57,7 @@ class Block(objects): self.ops.prepend(...) ``` -Users are able to create a global variable inside any block since they many create parameters inside a RNN or IfElseOp. All parameters should be stored in the global block, not the step block in RNN. +Users are able to create a global variable inside any block since they many create parameters inside a RNN or IfElse. All parameters should be stored in the global block, not the step block in RNN. Users can create local variables for outputs of operators. Users can also append and prepend an operator in current block. Prepending `random initialize` operator or `load` operator is very useful to initialize parameters before training. @@ -147,7 +147,7 @@ def fc_layer(input, size, block=None, ...): block = g_block w = block.create_parameter(...) b = block.create_parameter(...) - out = stack.create_var() + out = block.create_var() op = block.append_operator(Operator("FC", X=input, W=w, b=b, Out=out)) out.op = op return out From 531455651b720e8bd3b8eabf7b461e8d4138c3a9 Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Wed, 20 Sep 2017 17:40:15 -0700 Subject: [PATCH 014/170] Add `Program` to Python API design doc --- doc/design/python_api.md | 56 ++++++++++++++++++++++++++++++---------- 1 file changed, 43 insertions(+), 13 deletions(-) diff --git a/doc/design/python_api.md b/doc/design/python_api.md index 74d4d3239a..3122f5bfd8 100644 --- a/doc/design/python_api.md +++ b/doc/design/python_api.md @@ -17,24 +17,55 @@ Since we design our Python API concepts based on `compile-time`, we try to map o | Python Class | Compile-time protobuf | | --- | --- | +| Program | ProgramDesc | | Block | BlockDesc | | Operator | OpDesc | | Variable | VarDesc | +### Program + +`Program` is the description of the whole training process and there can only be one `Program` object, which is created automatically by the system at the very beginning. `Program` is formed by a series of `Block`. + +```python +class Program(objects): + def __init__(self): + self.blocks = vector() + self.blocks.append(Block(None)) + self.current_block_idx = 0 + + def get_block(block_idx): + return self.blocks[block_idx] + + def current_block(): + return self.get_block(self.current_block_idx) + + def fallback_current_block(): + self.current_block_idx = self.current_block().parent_idx + + + def create_block(): + new_block_idx = len(self.block) + self.blocks.append(Block(parent_idx=self.current_block_idx, + idx=new_block_idx)) + self.current_block_idx = new_block_idx +``` + +`Program` will create the first block in its constructor. The first block is called 'global block'. It is where all parameters are stored. ### Block Block is just like programming languages `{}`, which contains many operators and variables. There are two data fields in `Block`. 1) An associate map, whose key is variable name and value is variable itself; 2) A list of operators. -The block is hierarchical because PaddlePaddle supports RNN and IfElse. For example, RNN is like `for-loop` in programming languages. There is new `block` inside a `for-loop`. To represent hierarchies, `Block` stores the `parent Block` inside. If `parent=None`, the `Block` is the outermost block, i.e., the `global` block. +The block is hierarchical because PaddlePaddle supports RNN and IfElse. For example, RNN is like `for-loop` in programming languages. There is new `block` inside a `for-loop`. To represent hierarchies, `Block` stores the index of `parent Block` inside. The 'index' means the block's position in `Program`'s `blocks`. If `parent_idx=None`, the block itself is the outermost block, i.e., the 'global block'. ```python class Block(objects): - def __init__(self, parent=None): + def __init__(self, parent_idx, idx): self.vars = map() self.ops = vector() - self.parent = parent + self.idx = idx + self.parent_idx = parent_idx def create_var(self, ...): # create variable in `self.vars` @@ -42,8 +73,9 @@ class Block(objects): def create_global_var(self, ...): - if self.parent is not None: - return self.parent.create_global_var(...) + if self.parent_idx is not None: + parent_block = program.get_block(parent_idx) + return parent_block.create_global_var(...) else: return self.create_var(...) @@ -126,9 +158,8 @@ Here are examples of how to write a data layer and FC layer: ### Data Layer ```python -def data_layer(name, type, block=None): - if block is None: - block = g_block +def data_layer(name, type): + block = program.current_block() # type = dense_vector(size=10) / integer_value(range=10) return block.create_global_var( name=name, @@ -137,14 +168,13 @@ def data_layer(name, type, block=None): ``` -Before building new variables, we need to specify which block to use. If we don't, the default one `g_block` will be used. In the above `data_layer` code, a variable is created and be inserted into the root block to make it global. This variable is going to be used as input data of the whole network. +All the new variables and operators will be built in the `current block`. In the above `data_layer` code, a variable is created and be inserted into the root block to make it global. This variable is going to be used as input data of the whole network. ### FC Layer ```python -def fc_layer(input, size, block=None, ...): - if block is None: - block = g_block +def fc_layer(input, size, ...): + block = program.current_block() w = block.create_parameter(...) b = block.create_parameter(...) out = block.create_var() @@ -153,4 +183,4 @@ def fc_layer(input, size, block=None, ...): return out ``` -In the `fc_layer` code, we create two parameters(`w` and `b`), one variable(`out`) and one operator(`FC operator`), then insert all of them into the specified block. +In the `fc_layer` code, we create two parameters(`w` and `b`), one variable(`out`) and one operator(`FC operator`), then insert all of them into the `current block`. From a8477381ac87bb7369dfbd540011e1bbd5264998 Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Wed, 20 Sep 2017 16:26:24 +0800 Subject: [PATCH 015/170] Add pooling2d(max, ave) and pooling3d(max, ave) Op --- paddle/operators/CMakeLists.txt | 7 ++ paddle/operators/math/CMakeLists.txt | 7 +- paddle/operators/math/pooling.cc | 15 +-- paddle/operators/math/pooling.cu | 11 +- paddle/operators/pool_op.cc | 165 +++++++++++++++++++++++++++ paddle/operators/pool_op.cu | 26 +++++ paddle/operators/pool_op.h | 157 +++++++++++++++++++++++++ 7 files changed, 372 insertions(+), 16 deletions(-) create mode 100644 paddle/operators/pool_op.cc create mode 100644 paddle/operators/pool_op.cu create mode 100644 paddle/operators/pool_op.h diff --git a/paddle/operators/CMakeLists.txt b/paddle/operators/CMakeLists.txt index e3e934bccc..429e5526db 100644 --- a/paddle/operators/CMakeLists.txt +++ b/paddle/operators/CMakeLists.txt @@ -55,6 +55,13 @@ function(op_library TARGET) set(pybind_flag 1) endif() + # activation_op contains several operators + if ("${TARGET}" STREQUAL "pool_op") + set(pybind_flag 1) + # It's enough to just adding one operator to pybind + file(APPEND ${pybind_file} "USE_OP(pool2d);\n") + endif() + # pybind USE_NO_KERNEL_OP file(READ ${TARGET}.cc TARGET_CONTENT) string(REGEX MATCH "OperatorWithKernel" regex_result "${TARGET_CONTENT}") diff --git a/paddle/operators/math/CMakeLists.txt b/paddle/operators/math/CMakeLists.txt index f8333f34f7..185708cdaa 100644 --- a/paddle/operators/math/CMakeLists.txt +++ b/paddle/operators/math/CMakeLists.txt @@ -1,9 +1,8 @@ - if(WITH_GPU) - nv_library(math_function SRCS math_function.cc math_function.cu im2col.cc - im2col.cu DEPS cblas device_context) + nv_library(math_function SRCS math_function.cc math_function.cu im2col.cc + im2col.cu pooling.cc pooling.cu DEPS cblas device_context) else() - cc_library(math_function SRCS math_function.cc im2col.cc DEPS cblas device_context) + cc_library(math_function SRCS math_function.cc im2col.cc pooling.cc DEPS cblas device_context) endif() nv_test(math_function_test SRCS math_function_test.cc DEPS math_function tensor) diff --git a/paddle/operators/math/pooling.cc b/paddle/operators/math/pooling.cc index 6146e710fc..a78c5f929c 100644 --- a/paddle/operators/math/pooling.cc +++ b/paddle/operators/math/pooling.cc @@ -111,6 +111,7 @@ class Pool2dBackwardFunctor { int wend = std::min(wstart + ksize_width, input_width); wstart = std::max(wstart, 0); int pool_size = (hend - hstart) * (wend - wstart); + float scale = 1.0 / pool_size; for (int h = hstart; h < hend; ++h) { for (int w = wstart; w < wend; ++w) { pool_process.gradProcess( @@ -118,7 +119,7 @@ class Pool2dBackwardFunctor { output_data[ph * output_width + pw], output_grad_data[ph * output_width + pw], input_grad_data[h * input_width + w], - static_cast(pool_size)); + static_cast(scale)); } } } @@ -244,7 +245,6 @@ class Pool3dBackwardFunctor { const int padding_depth = paddings[0]; const int padding_height = paddings[1]; const int padding_width = paddings[2]; - const int input_stride = input_depth * input_height * input_width; const int output_stride = output_depth * output_height * output_width; @@ -271,6 +271,7 @@ class Pool3dBackwardFunctor { int pool_size = (dend - dstart) * (hend - hstart) * (wend - wstart); + float scale = 1.0 / pool_size; for (int d = dstart; d < dend; ++d) { for (int h = hstart; h < hend; ++h) { for (int w = wstart; w < wend; ++w) { @@ -280,17 +281,17 @@ class Pool3dBackwardFunctor { pool_process.gradProcess( input_data[input_idx], output_data[output_idx], output_grad_data[output_idx], - input_grad_data[input_idx], static_cast(pool_size)); + input_grad_data[input_idx], static_cast(scale)); } } } } } - input_data += input_stride; - output_data += output_stride; - input_grad_data += input_stride; - output_grad_data += output_stride; } + input_data += input_stride; + output_data += output_stride; + input_grad_data += input_stride; + output_grad_data += output_stride; } } } diff --git a/paddle/operators/math/pooling.cu b/paddle/operators/math/pooling.cu index 87bfb43c47..0a399f7ca0 100644 --- a/paddle/operators/math/pooling.cu +++ b/paddle/operators/math/pooling.cu @@ -95,7 +95,7 @@ __global__ void KernelPool2dBackward( int output_sub_idx = ph * output_width + pw; pool_process.gradProcess(input, output_data[output_sub_idx], output_grad[output_sub_idx], gradient, - static_cast(pool_size)); + static_cast(1.0 / pool_size)); } } input_grad[index] = gradient; @@ -264,7 +264,7 @@ __global__ void KernelPool3DBackward( int pdstart = (offsetD < ksize_depth) ? 0 - : (offsetD + ksize_depth) / stride_depth + 1; + : (offsetD - ksize_depth) / stride_depth + 1; int phstart = (offsetH < ksize_height) ? 0 : (offsetH - ksize_height) / stride_height + 1; @@ -296,10 +296,10 @@ __global__ void KernelPool3DBackward( hstart = max(hstart, 0); wstart = max(wstart, 0); int pool_size = (dend - dstart) * (hend - hstart) * (wend - wstart); - int output_sub_idx = ph * output_width + pw; + int output_sub_idx = (pd * output_height + ph) * output_width + pw; pool_process.gradProcess(input, output_data[output_sub_idx], output_grad[output_sub_idx], gradient, - static_cast(pool_size)); + static_cast(1.0 / pool_size)); } } } @@ -385,7 +385,8 @@ class Pool3dBackwardFunctor { const T* output_grad_data = output_grad.data(); T* input_grad_data = input_grad.mutable_data(context->GetPlace()); - int nthreads = batch_size * input_channels * input_height * input_width; + int nthreads = + batch_size * input_channels * input_depth * input_height * input_width; int blocks = (nthreads + 1024 - 1) / 1024; dim3 threads(1024, 1); dim3 grid(blocks, 1); diff --git a/paddle/operators/pool_op.cc b/paddle/operators/pool_op.cc new file mode 100644 index 0000000000..984cbefabf --- /dev/null +++ b/paddle/operators/pool_op.cc @@ -0,0 +1,165 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/operators/pool_op.h" + +namespace paddle { +namespace operators { + +int outputSize(int input_size, int filter_size, int padding, int stride) { + int output_size = (input_size - filter_size + 2 * padding) / stride + 1; + return output_size; +} + +class PoolOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(const framework::InferShapeContext &ctx) const override { + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Input"), + "Input(Input) of Pooling should not be null."); + PADDLE_ENFORCE_NOT_NULL(ctx.OutputVar("Output"), + "Output(Output) of Pooling should not be null."); + // PADDLE_ENFORCE_NOT_NULL(Attr("pooling_type"), + // "pooling_type should not be null."); + // PADDLE_ENFORCE_NOT_NULL(Attr>("ksize"), "ksize should + // not be null."); + auto input = ctx.Input("Input"); + auto output = ctx.Output("Output"); + int global_pooling = Attr("global_pooling"); + std::string pooling_type = Attr("pooling_type"); + std::vector ksize = Attr>("ksize"); + std::vector strides = Attr>("strides"); + std::vector paddings = Attr>("paddings"); + + PADDLE_ENFORCE(pooling_type == "max" || pooling_type == "ave", + "pooling_type should be 'max' or 'ave'"); + PADDLE_ENFORCE(ksize.size() == 2 || ksize.size() == 3, + "Pooling ksize should be 2-D or 3-D"); + + if (global_pooling == 1) { + for (size_t i = 0; i < ksize.size(); ++i) ksize[i] = input->dims()[i + 2]; + } + if (ksize.size() == 2) { + PADDLE_ENFORCE_EQ(input->dims().size(), 4, + "Pool2DOp intput should be 4-D."); + PADDLE_ENFORCE_EQ(strides.size(), 2, "Pool2DOp strides should be 2-D."); + PADDLE_ENFORCE_EQ(paddings.size(), 2, "Pool2DOp paddings should be 2-D."); + } else { + PADDLE_ENFORCE_EQ(input->dims().size(), 5, + "Pool3DOp intput should be 5-D."); + PADDLE_ENFORCE_EQ(strides.size(), 3, "Pool3DOp strides should be 3-D."); + PADDLE_ENFORCE_EQ(paddings.size(), 3, "Pool3DOp paddings should be 3-D."); + } + std::vector output_shape({input->dims()[0], input->dims()[1]}); + for (size_t i = 0; i < ksize.size(); ++i) { + output_shape.push_back( + outputSize(input->dims()[i + 2], ksize[i], paddings[i], strides[i])); + } + output->Resize(framework::make_ddim(output_shape)); + } +}; + +class PoolOpGrad : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(const framework::InferShapeContext &ctx) const override { + auto in = ctx.Input("Input"); + auto d_in = + ctx.Output(framework::GradVarName("Input")); + if (d_in) d_in->Resize(in->dims()); + } +}; + +class Pool3dOpMaker : public framework::OpProtoAndCheckerMaker { + public: + Pool3dOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput( + "Input", + "The input tensor of pooling operator. " + "The format of input tensor is NCDHW. Where N is batch size, C is the " + "number of channels, D, H and W is the depth, height and width of " + "image."); + AddOutput("Output", + "The output tensor of pooling operator." + "The format of output tensor is also NCDHW."); + + AddAttr("pooling_type", + "pooling_type of pooling operator.['max' or 'ave']"); + AddAttr>("ksize", "strides of pooling operator."); + AddAttr("global_pooling", "whether to use the global_pooling.") + .SetDefault(0); + AddAttr>("strides", "strides of pooling operator.") + .SetDefault({1, 1, 1}); + AddAttr>("paddings", "paddings of pooling operator.") + .SetDefault({0, 0, 0}); + AddComment(R"DOC( +The pooling3d operation calculates the output based on +the input, pooling_type and ksize, strides, paddings parameters. +)DOC"); + } +}; + +class Pool2dOpMaker : public framework::OpProtoAndCheckerMaker { + public: + Pool2dOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput( + "Input", + "The input tensor of pooling operator. " + "The format of input tensor is NCHW. Where N is batch size, C is the " + "number of channels, H and W is the height and width of image."); + AddOutput("Output", + "The output tensor of pooling operator." + "The format of output tensor is also NCHW."); + + AddAttr("pooling_type", + "pooling_type of pooling operator.['max' or 'ave']"); + AddAttr>("ksize", "strides of pooling operator."); + AddAttr("global_pooling", "whether to use the global_pooling.") + .SetDefault(0); + AddAttr>("strides", "strides of pooling operator.") + .SetDefault({1, 1}); + AddAttr>("paddings", "paddings of pooling operator.") + .SetDefault({0, 0}); + AddComment(R"DOC( +The pooling2d operation calculates the output based on +the input, pooling_type and ksize, strides, paddings parameters. +)DOC"); + } +}; +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; + +REGISTER_OP(pool2d, ops::PoolOp, ops::Pool2dOpMaker, pool2d_grad, + ops::PoolOpGrad); + +REGISTER_OP_CPU_KERNEL(pool2d, + ops::PoolKernel); +REGISTER_OP_CPU_KERNEL(pool2d_grad, + ops::PoolGradKernel) + +REGISTER_OP(pool3d, ops::PoolOp, ops::Pool3dOpMaker, pool3d_grad, + ops::PoolOpGrad); + +REGISTER_OP_CPU_KERNEL(pool3d, + ops::PoolKernel); +REGISTER_OP_CPU_KERNEL(pool3d_grad, + ops::PoolGradKernel); diff --git a/paddle/operators/pool_op.cu b/paddle/operators/pool_op.cu new file mode 100644 index 0000000000..b011d20da5 --- /dev/null +++ b/paddle/operators/pool_op.cu @@ -0,0 +1,26 @@ +/* Copyright (c) 2016 PaddlePaddle Authors All Rights Reserve. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/operators/pool_op.h" + +namespace ops = paddle::operators; + +REGISTER_OP_GPU_KERNEL(pool2d, + ops::PoolKernel); +REGISTER_OP_GPU_KERNEL(pool2d_grad, + ops::PoolGradKernel); + +REGISTER_OP_GPU_KERNEL(pool3d, + ops::PoolKernel); +REGISTER_OP_GPU_KERNEL(pool3d_grad, + ops::PoolGradKernel); diff --git a/paddle/operators/pool_op.h b/paddle/operators/pool_op.h new file mode 100644 index 0000000000..aca1c5a137 --- /dev/null +++ b/paddle/operators/pool_op.h @@ -0,0 +1,157 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include "paddle/framework/eigen.h" +#include "paddle/framework/op_registry.h" +#include "paddle/operators/math/math_function.h" +#include "paddle/operators/math/pooling.h" + +namespace paddle { +namespace operators { + +using Tensor = framework::Tensor; + +template +class PoolKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + const Tensor* input = context.Input("Input"); + Tensor* output = context.Output("Output"); + + int global_pooling = context.Attr("global_pooling"); + std::string pooling_type = context.Attr("pooling_type"); + std::vector ksize = context.Attr>("ksize"); + std::vector strides = context.Attr>("strides"); + std::vector paddings = context.Attr>("paddings"); + if (global_pooling == 1) { + for (size_t i = 0; i < ksize.size(); ++i) { + ksize[i] = input->dims()[i + 2]; + } + } + auto* device_context = + const_cast(context.device_context_); + + switch (ksize.size()) { + case 2: { + if (pooling_type == "max") { + paddle::operators::math::Pool2dForwardFunctor< + Place, paddle::operators::math::pool::maxPool, T> + pool2d_forward; + paddle::operators::math::pool::maxPool pool_process; + pool2d_forward(*input, *output, ksize, strides, paddings, + pool_process, device_context); + + } else if (pooling_type == "ave") { + paddle::operators::math::Pool2dForwardFunctor< + Place, paddle::operators::math::pool::avePool, T> + pool2d_forward; + paddle::operators::math::pool::avePool pool_process; + pool2d_forward(*input, *output, ksize, strides, paddings, + pool_process, device_context); + } + } break; + case 3: { + if (pooling_type == "max") { + paddle::operators::math::Pool3dForwardFunctor< + Place, paddle::operators::math::pool::maxPool, T> + pool3d_forward; + paddle::operators::math::pool::maxPool pool_process; + pool3d_forward(*input, *output, ksize, strides, paddings, + pool_process, device_context); + } else if (pooling_type == "ave") { + paddle::operators::math::Pool3dForwardFunctor< + Place, paddle::operators::math::pool::avePool, T> + pool3d_forward; + paddle::operators::math::pool::avePool pool_process; + pool3d_forward(*input, *output, ksize, strides, paddings, + pool_process, device_context); + } + } break; + } + } +}; + +template +class PoolGradKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& context) const override { + const Tensor* input = context.Input("Input"); + const Tensor* output = context.Input("Output"); + const Tensor* output_grad = + context.Input(framework::GradVarName("Output")); + Tensor* input_grad = + context.Output(framework::GradVarName("Input")); + + int global_pooling = context.Attr("global_pooling"); + std::string pooling_type = context.Attr("pooling_type"); + std::vector ksize = context.Attr>("ksize"); + std::vector strides = context.Attr>("strides"); + std::vector paddings = context.Attr>("paddings"); + + if (global_pooling == 1) { + for (size_t i = 0; i < ksize.size(); ++i) ksize[i] = input->dims()[i + 2]; + } + auto* device_context = + const_cast(context.device_context_); + + if (input_grad) { + input_grad->mutable_data(context.GetPlace()); + auto temp = framework::EigenVector::Flatten(*input_grad); + temp.device(context.GetEigenDevice()) = + temp.constant(static_cast(0)); + + switch (ksize.size()) { + case 2: { + if (pooling_type == "max") { + paddle::operators::math::Pool2dBackwardFunctor< + Place, paddle::operators::math::pool::maxPool, T> + pool2d_backward; + paddle::operators::math::pool::maxPool pool_process; + pool2d_backward(*input, *input_grad, *output, *output_grad, ksize, + strides, paddings, pool_process, device_context); + } else if (pooling_type == "ave") { + paddle::operators::math::Pool2dBackwardFunctor< + Place, paddle::operators::math::pool::avePool, T> + pool2d_backward; + paddle::operators::math::pool::avePool pool_process; + pool2d_backward(*input, *input_grad, *output, *output_grad, ksize, + strides, paddings, pool_process, device_context); + } + } break; + case 3: { + if (pooling_type == "max") { + paddle::operators::math::Pool3dBackwardFunctor< + Place, paddle::operators::math::pool::maxPool, T> + pool3d_backward; + paddle::operators::math::pool::maxPool pool_process; + pool3d_backward(*input, *input_grad, *output, *output_grad, ksize, + strides, paddings, pool_process, device_context); + } else if (pooling_type == "ave") { + paddle::operators::math::Pool3dBackwardFunctor< + Place, paddle::operators::math::pool::avePool, T> + pool3d_backward; + paddle::operators::math::pool::avePool pool_process; + pool3d_backward(*input, *input_grad, *output, *output_grad, ksize, + strides, paddings, pool_process, device_context); + } + } break; + } + } + } +}; + +} // namespace operators +} // namespace paddle From 33d99998900950e114f8d6d16496f59d534de976 Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Thu, 21 Sep 2017 15:48:23 +0800 Subject: [PATCH 016/170] Add pool2d unit test --- .../v2/framework/tests/test_pool2d_op.py | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 python/paddle/v2/framework/tests/test_pool2d_op.py diff --git a/python/paddle/v2/framework/tests/test_pool2d_op.py b/python/paddle/v2/framework/tests/test_pool2d_op.py new file mode 100644 index 0000000000..549898ec94 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_pool2d_op.py @@ -0,0 +1,98 @@ +import unittest +import numpy as np +from op_test import OpTest + + +def max_pool2D_forward_naive(x, ksize, strides, paddings=[0, 0]): + + N, C, H, W = x.shape + H_out = (H - ksize[0] + 2 * paddings[0]) / strides[0] + 1 + W_out = (W - ksize[1] + 2 * paddings[1]) / strides[1] + 1 + out = np.zeros((N, C, H_out, W_out)) + for i in xrange(H_out): + for j in xrange(W_out): + r_start = np.max((i * strides[0] - paddings[0], 0)) + r_end = np.min((i * strides[0] + ksize[0] - paddings[0], H)) + c_start = np.max((j * strides[1] - paddings[1], 0)) + c_end = np.min((j * strides[1] + ksize[1] - paddings[1], W)) + x_masked = x[:, :, r_start:r_end, c_start:c_end] + + out[:, :, i, j] = np.max(x_masked, axis=(2, 3)) + return out + + +def ave_pool2D_forward_naive(x, ksize, strides, paddings=[0, 0]): + + N, C, H, W = x.shape + H_out = (H - ksize[0] + 2 * paddings[0]) / strides[0] + 1 + W_out = (W - ksize[1] + 2 * paddings[1]) / strides[1] + 1 + out = np.zeros((N, C, H_out, W_out)) + for i in xrange(H_out): + for j in xrange(W_out): + r_start = np.max((i * strides[0] - paddings[0], 0)) + r_end = np.min((i * strides[0] + ksize[0] - paddings[0], H)) + c_start = np.max((j * strides[1] - paddings[1], 0)) + c_end = np.min((j * strides[1] + ksize[1] - paddings[1], W)) + x_masked = x[:, :, r_start:r_end, c_start:c_end] + + out[:, :, i, j] = np.sum(x_masked, axis=(2, 3)) / ( + (r_end - r_start) * (c_end - c_start)) + return out + + +class TestPool2d_Op(OpTest): + def setUp(self): + self.initTestCase() + self.op_type = "pool2d" + input = np.random.random(self.shape).astype("float32") + output = self.pool2D_forward_naive(input, self.ksize, self.strides, + self.paddings) + self.inputs = {'Input': input} + + self.attrs = { + 'strides': self.strides, + 'paddings': self.paddings, + 'ksize': self.ksize, + 'pooling_type': self.pool_type, + } + + self.outputs = {'Output': output} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(set(['Input']), 'Output', max_relative_error=0.07) + + def initTestCase(self): + self.pool_type = "ave" + self.pool2D_forward_naive = ave_pool2D_forward_naive + self.shape = [2, 3, 5, 5] + self.ksize = [3, 3] + self.strides = [1, 1] + self.paddings = [0, 0] + + +class TestCase2(TestPool2d_Op): + def initTestCase(self): + self.op_type = "pool2d" + self.pool_type = "ave" + self.pool2D_forward_naive = ave_pool2D_forward_naive + self.shape = [2, 3, 5, 5] + self.ksize = [3, 3] + self.strides = [1, 1] + self.paddings = [1, 1] + + +# class TestCase1(TestPool2d_Op): +# def initTestCase(self): +# self.op_type = "pool2d" +# self.pool_type = "max" +# self.pool2D_forward_naive = max_pool2D_forward_naive +# self.shape = [2, 3, 5, 5] +# self.ksize = [3, 3] +# self.strides = [1, 1] +# self.paddings = [1, 1] + +if __name__ == '__main__': + unittest.main() From 510f00800a43c8df18ac4551d2b35fba0a400d5d Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Thu, 21 Sep 2017 15:49:16 +0800 Subject: [PATCH 017/170] Add pool3d unit test --- .../v2/framework/tests/test_pool3d_op.py | 108 ++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 python/paddle/v2/framework/tests/test_pool3d_op.py diff --git a/python/paddle/v2/framework/tests/test_pool3d_op.py b/python/paddle/v2/framework/tests/test_pool3d_op.py new file mode 100644 index 0000000000..f8e9a768e0 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_pool3d_op.py @@ -0,0 +1,108 @@ +import unittest +import numpy as np +from op_test import OpTest + + +def max_pool3D_forward_naive(x, ksize, strides, paddings=[0, 0]): + + N, C, D, H, W = x.shape + D_out = (D - ksize[0] + 2 * paddings[0]) / strides[0] + 1 + H_out = (H - ksize[1] + 2 * paddings[1]) / strides[1] + 1 + W_out = (W - ksize[2] + 2 * paddings[2]) / strides[2] + 1 + out = np.zeros((N, C, D_out, H_out, W_out)) + for k in xrange(D_out): + d_start = np.max((k * strides[0] - paddings[0], 0)) + d_end = np.min((k * strides[0] + ksize[0] - paddings[0], D)) + for i in xrange(H_out): + h_start = np.max((i * strides[0] - paddings[0], 0)) + h_end = np.min((i * strides[0] + ksize[0] - paddings[0], H)) + for j in xrange(W_out): + w_start = np.max((j * strides[1] - paddings[1], 0)) + w_end = np.min((j * strides[1] + ksize[1] - paddings[1], W)) + + x_masked = x[:, :, d_start:d_end, h_start:h_end, w_start:w_end] + + out[:, :, k, i, j] = np.max(x_masked, axis=(2, 3, 4)) + return out + + +def ave_pool3D_forward_naive(x, ksize, strides, paddings=[0, 0]): + + N, C, D, H, W = x.shape + D_out = (D - ksize[0] + 2 * paddings[0]) / strides[0] + 1 + H_out = (H - ksize[1] + 2 * paddings[1]) / strides[1] + 1 + W_out = (W - ksize[2] + 2 * paddings[2]) / strides[2] + 1 + out = np.zeros((N, C, D_out, H_out, W_out)) + for k in xrange(D_out): + d_start = np.max((k * strides[0] - paddings[0], 0)) + d_end = np.min((k * strides[0] + ksize[0] - paddings[0], D)) + for i in xrange(H_out): + h_start = np.max((i * strides[0] - paddings[0], 0)) + h_end = np.min((i * strides[0] + ksize[0] - paddings[0], H)) + for j in xrange(W_out): + w_start = np.max((j * strides[1] - paddings[1], 0)) + w_end = np.min((j * strides[1] + ksize[1] - paddings[1], W)) + + x_masked = x[:, :, d_start:d_end, h_start:h_end, w_start:w_end] + + out[:, :, k, i, j] = np.sum(x_masked, axis=(2, 3, 4)) / ( + (d_end - d_start) * (h_end - h_start) * (w_end - w_start)) + return out + + +class TestPool3d_Op(OpTest): + def setUp(self): + self.initTestCase() + self.op_type = "pool3d" + input = np.random.random(self.shape).astype("float32") + output = self.pool3D_forward_naive(input, self.ksize, self.strides, + self.paddings) + self.inputs = {'Input': input} + + self.attrs = { + 'strides': self.strides, + 'paddings': self.paddings, + 'ksize': self.ksize, + 'pooling_type': self.pool_type, + } + + self.outputs = {'Output': output} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(set(['Input']), 'Output', max_relative_error=0.07) + + def initTestCase(self): + self.pool_type = "ave" + self.pool3D_forward_naive = ave_pool3D_forward_naive + self.shape = [2, 3, 5, 5, 5] + self.ksize = [3, 3, 3] + self.strides = [1, 1, 1] + self.paddings = [0, 0, 0] + + +class TestCase1(TestPool3d_Op): + def initTestCase(self): + self.op_type = "pool3d" + self.pool_type = "ave" + self.pool3D_forward_naive = ave_pool3D_forward_naive + self.shape = [2, 3, 7, 7, 7] + self.ksize = [3, 3, 3] + self.strides = [1, 1, 1] + self.paddings = [1, 1, 1] + + +# class TestCase2(TestPool3d_Op): +# def initTestCase(self): +# self.op_type = "pool3d" +# self.pool_type = "max" +# self.pool3D_forward_naive = max_pool3D_forward_naive +# self.shape = [2, 3, 5, 5, 5] +# self.ksize = [3, 3, 3] +# self.strides = [1, 1, 1] +# self.paddings = [1, 1, 1] + +if __name__ == '__main__': + unittest.main() From 50b8ec0564534b07fe5fdacd91db6a21fafd0559 Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Thu, 21 Sep 2017 19:21:12 +0800 Subject: [PATCH 018/170] fix unit test --- .../v2/framework/tests/test_pool2d_op.py | 22 ++++++++++--------- .../v2/framework/tests/test_pool3d_op.py | 22 ++++++++++--------- 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/python/paddle/v2/framework/tests/test_pool2d_op.py b/python/paddle/v2/framework/tests/test_pool2d_op.py index 549898ec94..cf327508f7 100644 --- a/python/paddle/v2/framework/tests/test_pool2d_op.py +++ b/python/paddle/v2/framework/tests/test_pool2d_op.py @@ -62,7 +62,8 @@ class TestPool2d_Op(OpTest): self.check_output() def test_check_grad(self): - self.check_grad(set(['Input']), 'Output', max_relative_error=0.07) + if self.pool_type != "max": + self.check_grad(set(['Input']), 'Output', max_relative_error=0.07) def initTestCase(self): self.pool_type = "ave" @@ -84,15 +85,16 @@ class TestCase2(TestPool2d_Op): self.paddings = [1, 1] -# class TestCase1(TestPool2d_Op): -# def initTestCase(self): -# self.op_type = "pool2d" -# self.pool_type = "max" -# self.pool2D_forward_naive = max_pool2D_forward_naive -# self.shape = [2, 3, 5, 5] -# self.ksize = [3, 3] -# self.strides = [1, 1] -# self.paddings = [1, 1] +class TestCase1(TestPool2d_Op): + def initTestCase(self): + self.op_type = "pool2d" + self.pool_type = "max" + self.pool2D_forward_naive = max_pool2D_forward_naive + self.shape = [2, 3, 5, 5] + self.ksize = [3, 3] + self.strides = [1, 1] + self.paddings = [1, 1] + if __name__ == '__main__': unittest.main() diff --git a/python/paddle/v2/framework/tests/test_pool3d_op.py b/python/paddle/v2/framework/tests/test_pool3d_op.py index f8e9a768e0..cfd0ced150 100644 --- a/python/paddle/v2/framework/tests/test_pool3d_op.py +++ b/python/paddle/v2/framework/tests/test_pool3d_op.py @@ -72,7 +72,8 @@ class TestPool3d_Op(OpTest): self.check_output() def test_check_grad(self): - self.check_grad(set(['Input']), 'Output', max_relative_error=0.07) + if self.pool_type != "max": + self.check_grad(set(['Input']), 'Output', max_relative_error=0.07) def initTestCase(self): self.pool_type = "ave" @@ -94,15 +95,16 @@ class TestCase1(TestPool3d_Op): self.paddings = [1, 1, 1] -# class TestCase2(TestPool3d_Op): -# def initTestCase(self): -# self.op_type = "pool3d" -# self.pool_type = "max" -# self.pool3D_forward_naive = max_pool3D_forward_naive -# self.shape = [2, 3, 5, 5, 5] -# self.ksize = [3, 3, 3] -# self.strides = [1, 1, 1] -# self.paddings = [1, 1, 1] +class TestCase2(TestPool3d_Op): + def initTestCase(self): + self.op_type = "pool3d" + self.pool_type = "max" + self.pool3D_forward_naive = max_pool3D_forward_naive + self.shape = [2, 3, 5, 5, 5] + self.ksize = [3, 3, 3] + self.strides = [1, 1, 1] + self.paddings = [1, 1, 1] + if __name__ == '__main__': unittest.main() From 84a2512b90f39854386fb03f437ac8a92a486437 Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Fri, 22 Sep 2017 10:06:03 +0800 Subject: [PATCH 019/170] fix parameter name and function define --- paddle/operators/math/pooling.cc | 24 ++--- paddle/operators/math/pooling.cu | 24 ++--- paddle/operators/math/pooling.h | 24 ++--- paddle/operators/pool_op.cc | 87 +++++++++++-------- paddle/operators/pool_op.h | 72 ++++++++------- .../v2/framework/tests/test_pool2d_op.py | 8 +- .../v2/framework/tests/test_pool3d_op.py | 8 +- 7 files changed, 129 insertions(+), 118 deletions(-) diff --git a/paddle/operators/math/pooling.cc b/paddle/operators/math/pooling.cc index 671bead1b4..5ce748ff08 100644 --- a/paddle/operators/math/pooling.cc +++ b/paddle/operators/math/pooling.cc @@ -21,10 +21,10 @@ namespace math { template class Pool2dForwardFunctor { public: - void operator()(const framework::Tensor& input, framework::Tensor& output, + void operator()(const platform::DeviceContext& context, + const framework::Tensor& input, framework::Tensor& output, std::vector& ksize, std::vector& strides, - std::vector& paddings, PoolProcess pool_process, - const platform::DeviceContext& context) { + std::vector& paddings, PoolProcess pool_process) { const int batch_size = input.dims()[0]; const int input_height = input.dims()[2]; const int input_width = input.dims()[3]; @@ -75,12 +75,12 @@ class Pool2dForwardFunctor { template class Pool2dBackwardFunctor { public: - void operator()(const framework::Tensor& input, framework::Tensor& input_grad, + void operator()(const platform::DeviceContext& context, + const framework::Tensor& input, framework::Tensor& input_grad, const framework::Tensor& output, const framework::Tensor& output_grad, std::vector& ksize, std::vector& strides, std::vector& paddings, - PoolProcess pool_process, - const platform::DeviceContext& context) { + PoolProcess pool_process) { const int batch_size = input.dims()[0]; const int input_height = input.dims()[2]; const int input_width = input.dims()[3]; @@ -154,10 +154,10 @@ template class Pool2dBackwardFunctor< template class Pool3dForwardFunctor { public: - void operator()(const framework::Tensor& input, framework::Tensor& output, + void operator()(const platform::DeviceContext& context, + const framework::Tensor& input, framework::Tensor& output, std::vector& ksize, std::vector& strides, - std::vector& paddings, PoolProcess pool_process, - const platform::DeviceContext& context) { + std::vector& paddings, PoolProcess pool_process) { const int batch_size = input.dims()[0]; const int input_depth = input.dims()[2]; const int input_height = input.dims()[3]; @@ -224,12 +224,12 @@ class Pool3dForwardFunctor { template class Pool3dBackwardFunctor { public: - void operator()(const framework::Tensor& input, framework::Tensor& input_grad, + void operator()(const platform::DeviceContext& context, + const framework::Tensor& input, framework::Tensor& input_grad, const framework::Tensor& output, const framework::Tensor& output_grad, std::vector& ksize, std::vector& strides, std::vector& paddings, - PoolProcess pool_process, - const platform::DeviceContext& context) { + PoolProcess pool_process) { const int batch_size = input.dims()[0]; const int input_depth = input.dims()[2]; const int input_height = input.dims()[3]; diff --git a/paddle/operators/math/pooling.cu b/paddle/operators/math/pooling.cu index ce0a01776a..124006942c 100644 --- a/paddle/operators/math/pooling.cu +++ b/paddle/operators/math/pooling.cu @@ -105,10 +105,10 @@ __global__ void KernelPool2dBackward( template class Pool2dForwardFunctor { public: - void operator()(const framework::Tensor& input, framework::Tensor& output, + void operator()(const platform::DeviceContext& context, + const framework::Tensor& input, framework::Tensor& output, std::vector& ksize, std::vector& strides, - std::vector& paddings, PoolProcess pool_process, - const platform::DeviceContext& context) { + std::vector& paddings, PoolProcess pool_process) { const int batch_size = input.dims()[0]; const int input_channels = input.dims()[1]; const int input_height = input.dims()[2]; @@ -148,12 +148,12 @@ class Pool2dForwardFunctor { template class Pool2dBackwardFunctor { public: - void operator()(const framework::Tensor& input, framework::Tensor& input_grad, + void operator()(const platform::DeviceContext& context, + const framework::Tensor& input, framework::Tensor& input_grad, const framework::Tensor& output, const framework::Tensor& output_grad, std::vector& ksize, std::vector& strides, std::vector& paddings, - PoolProcess pool_process, - const platform::DeviceContext& context) { + PoolProcess pool_process) { const int batch_size = input.dims()[0]; const int input_channels = input.dims()[1]; const int input_height = input.dims()[2]; @@ -319,10 +319,10 @@ __global__ void KernelPool3DBackward( template class Pool3dForwardFunctor { public: - void operator()(const framework::Tensor& input, framework::Tensor& output, + void operator()(const platform::DeviceContext& context, + const framework::Tensor& input, framework::Tensor& output, std::vector& ksize, std::vector& strides, - std::vector& paddings, PoolProcess pool_process, - const platform::DeviceContext& context) { + std::vector& paddings, PoolProcess pool_process) { const int batch_size = input.dims()[0]; const int input_channels = input.dims()[1]; const int input_depth = input.dims()[2]; @@ -369,12 +369,12 @@ class Pool3dForwardFunctor { template class Pool3dBackwardFunctor { public: - void operator()(const framework::Tensor& input, framework::Tensor& input_grad, + void operator()(const platform::DeviceContext& context, + const framework::Tensor& input, framework::Tensor& input_grad, const framework::Tensor& output, const framework::Tensor& output_grad, std::vector& ksize, std::vector& strides, std::vector& paddings, - PoolProcess pool_process, - const platform::DeviceContext& context) { + PoolProcess pool_process) { const int batch_size = input.dims()[0]; const int input_channels = input.dims()[1]; const int input_depth = input.dims()[2]; diff --git a/paddle/operators/math/pooling.h b/paddle/operators/math/pooling.h index aad5a1837b..3f01c9dacb 100644 --- a/paddle/operators/math/pooling.h +++ b/paddle/operators/math/pooling.h @@ -59,41 +59,41 @@ class avePool { template class Pool2dForwardFunctor { public: - void operator()(const framework::Tensor& input, framework::Tensor& output, + void operator()(const platform::DeviceContext& context, + const framework::Tensor& input, framework::Tensor& output, std::vector& ksize, std::vector& strides, - std::vector& paddings, PoolProcess pool_process, - const platform::DeviceContext& context); + std::vector& paddings, PoolProcess pool_process); }; template class Pool2dBackwardFunctor { public: - void operator()(const framework::Tensor& input, framework::Tensor& input_grad, + void operator()(const platform::DeviceContext& context, + const framework::Tensor& input, framework::Tensor& input_grad, const framework::Tensor& output, const framework::Tensor& output_grad, std::vector& ksize, std::vector& strides, std::vector& paddings, - PoolProcess pool_process, - const platform::DeviceContext& context); + PoolProcess pool_process); }; template class Pool3dForwardFunctor { public: - void operator()(const framework::Tensor& input, framework::Tensor& output, + void operator()(const platform::DeviceContext& context, + const framework::Tensor& input, framework::Tensor& output, std::vector& ksize, std::vector& strides, - std::vector& paddings, PoolProcess pool_process, - const platform::DeviceContext& context); + std::vector& paddings, PoolProcess pool_process); }; template class Pool3dBackwardFunctor { public: - void operator()(const framework::Tensor& input, framework::Tensor& input_grad, + void operator()(const platform::DeviceContext& context, + const framework::Tensor& input, framework::Tensor& input_grad, const framework::Tensor& output, const framework::Tensor& output_grad, std::vector& ksize, std::vector& strides, std::vector& paddings, - PoolProcess pool_process, - const platform::DeviceContext& context); + PoolProcess pool_process); }; } // namespace math diff --git a/paddle/operators/pool_op.cc b/paddle/operators/pool_op.cc index 1d79629d73..c23adf63bf 100644 --- a/paddle/operators/pool_op.cc +++ b/paddle/operators/pool_op.cc @@ -28,18 +28,18 @@ class PoolOp : public framework::OperatorWithKernel { protected: void InferShape(const framework::InferShapeContext &ctx) const override { - PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Input"), - "Input(Input) of Pooling should not be null."); - PADDLE_ENFORCE_NOT_NULL(ctx.OutputVar("Output"), - "Output(Output) of Pooling should not be null."); - // PADDLE_ENFORCE_NOT_NULL(Attr("pooling_type"), + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), + "X(Input) of Pooling should not be null."); + PADDLE_ENFORCE_NOT_NULL(ctx.OutputVar("Out"), + "Out(Output) of Pooling should not be null."); + // PADDLE_ENFORCE_NOT_NULL(Attr("poolingType"), // "pooling_type should not be null."); // PADDLE_ENFORCE_NOT_NULL(Attr>("ksize"), "ksize should // not be null."); - auto input = ctx.Input("Input"); - auto output = ctx.Output("Output"); - int global_pooling = Attr("global_pooling"); - std::string pooling_type = Attr("pooling_type"); + auto in_X = ctx.Input("X"); + auto out = ctx.Output("Out"); + int global_pooling = Attr("globalPooling"); + std::string pooling_type = Attr("poolingType"); std::vector ksize = Attr>("ksize"); std::vector strides = Attr>("strides"); std::vector paddings = Attr>("paddings"); @@ -50,25 +50,25 @@ class PoolOp : public framework::OperatorWithKernel { "Pooling ksize should be 2-D or 3-D"); if (global_pooling == 1) { - for (size_t i = 0; i < ksize.size(); ++i) ksize[i] = input->dims()[i + 2]; + for (size_t i = 0; i < ksize.size(); ++i) ksize[i] = in_X->dims()[i + 2]; } if (ksize.size() == 2) { - PADDLE_ENFORCE_EQ(input->dims().size(), 4, + PADDLE_ENFORCE_EQ(in_X->dims().size(), 4, "Pool2DOp intput should be 4-D."); PADDLE_ENFORCE_EQ(strides.size(), 2, "Pool2DOp strides should be 2-D."); PADDLE_ENFORCE_EQ(paddings.size(), 2, "Pool2DOp paddings should be 2-D."); } else { - PADDLE_ENFORCE_EQ(input->dims().size(), 5, + PADDLE_ENFORCE_EQ(in_X->dims().size(), 5, "Pool3DOp intput should be 5-D."); PADDLE_ENFORCE_EQ(strides.size(), 3, "Pool3DOp strides should be 3-D."); PADDLE_ENFORCE_EQ(paddings.size(), 3, "Pool3DOp paddings should be 3-D."); } - std::vector output_shape({input->dims()[0], input->dims()[1]}); + std::vector output_shape({in_X->dims()[0], in_X->dims()[1]}); for (size_t i = 0; i < ksize.size(); ++i) { - output_shape.push_back(outputSize_pool(input->dims()[i + 2], ksize[i], + output_shape.push_back(outputSize_pool(in_X->dims()[i + 2], ksize[i], paddings[i], strides[i])); } - output->Resize(framework::make_ddim(output_shape)); + out->Resize(framework::make_ddim(output_shape)); } }; @@ -78,9 +78,8 @@ class PoolOpGrad : public framework::OperatorWithKernel { protected: void InferShape(const framework::InferShapeContext &ctx) const override { - auto in = ctx.Input("Input"); - auto d_in = - ctx.Output(framework::GradVarName("Input")); + auto in = ctx.Input("X"); + auto d_in = ctx.Output(framework::GradVarName("X")); if (d_in) d_in->Resize(in->dims()); } }; @@ -90,27 +89,36 @@ class Pool3dOpMaker : public framework::OpProtoAndCheckerMaker { Pool3dOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { AddInput( - "Input", + "X", "The input tensor of pooling operator. " "The format of input tensor is NCDHW. Where N is batch size, C is the " "number of channels, D, H and W is the depth, height and width of " "image."); - AddOutput("Output", + AddOutput("Out", "The output tensor of pooling operator." "The format of output tensor is also NCDHW."); - AddAttr("pooling_type", - "pooling_type of pooling operator.['max' or 'ave']"); - AddAttr>("ksize", "strides of pooling operator."); - AddAttr("global_pooling", "whether to use the global_pooling.") + AddAttr("poolingType", + "poolingType of pooling operator.['max' or 'ave']"); + AddAttr>( + "ksize", "pooling size(depth, height, width) of pooling operator."); + AddAttr("globalPooling", + "default 0" + "whether to use the globalPooling.") .SetDefault(0); - AddAttr>("strides", "strides of pooling operator.") + AddAttr>( + "strides", + "default {1,1,1}" + "strides(depth, height, width) of pooling operator.") .SetDefault({1, 1, 1}); - AddAttr>("paddings", "paddings of pooling operator.") + AddAttr>( + "paddings", + "default {0,0,0}" + "paddings(depth, height, width) of pooling operator.") .SetDefault({0, 0, 0}); AddComment(R"DOC( The pooling3d operation calculates the output based on -the input, pooling_type and ksize, strides, paddings parameters. +the input, poolingType and ksize, strides, paddings parameters. )DOC"); } }; @@ -120,26 +128,33 @@ class Pool2dOpMaker : public framework::OpProtoAndCheckerMaker { Pool2dOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { AddInput( - "Input", + "X", "The input tensor of pooling operator. " "The format of input tensor is NCHW. Where N is batch size, C is the " "number of channels, H and W is the height and width of image."); - AddOutput("Output", + AddOutput("Out", "The output tensor of pooling operator." "The format of output tensor is also NCHW."); - AddAttr("pooling_type", - "pooling_type of pooling operator.['max' or 'ave']"); - AddAttr>("ksize", "strides of pooling operator."); - AddAttr("global_pooling", "whether to use the global_pooling.") + AddAttr("poolingType", + "poolingType of pooling operator.['max' or 'ave']"); + AddAttr>( + "ksize", "pooling size(height, width) of pooling operator."); + AddAttr("globalPooling", + "default 0" + "whether to use the globalPooling.[0 or 1]") .SetDefault(0); - AddAttr>("strides", "strides of pooling operator.") + AddAttr>("strides", + "default {1, 1}" + "strides(height, width) of pooling operator.") .SetDefault({1, 1}); - AddAttr>("paddings", "paddings of pooling operator.") + AddAttr>("paddings", + "default {0, 0}" + "paddings(height, width) of pooling operator.") .SetDefault({0, 0}); AddComment(R"DOC( The pooling2d operation calculates the output based on -the input, pooling_type and ksize, strides, paddings parameters. +the input, poolingType and ksize, strides, paddings parameters. )DOC"); } }; diff --git a/paddle/operators/pool_op.h b/paddle/operators/pool_op.h index 16779cbb91..2e737f0a4b 100644 --- a/paddle/operators/pool_op.h +++ b/paddle/operators/pool_op.h @@ -28,17 +28,17 @@ template class PoolKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { - const Tensor* input = context.Input("Input"); - Tensor* output = context.Output("Output"); + const Tensor* in_X = context.Input("X"); + Tensor* out = context.Output("Out"); - int global_pooling = context.Attr("global_pooling"); - std::string pooling_type = context.Attr("pooling_type"); + int global_pooling = context.Attr("globalPooling"); + std::string pooling_type = context.Attr("poolingType"); std::vector ksize = context.Attr>("ksize"); std::vector strides = context.Attr>("strides"); std::vector paddings = context.Attr>("paddings"); if (global_pooling == 1) { for (size_t i = 0; i < ksize.size(); ++i) { - ksize[i] = input->dims()[i + 2]; + ksize[i] = in_X->dims()[i + 2]; } } @@ -49,16 +49,16 @@ class PoolKernel : public framework::OpKernel { Place, paddle::operators::math::pool::maxPool, T> pool2d_forward; paddle::operators::math::pool::maxPool pool_process; - pool2d_forward(*input, *output, ksize, strides, paddings, - pool_process, context.device_context()); + pool2d_forward(context.device_context(), *in_X, *out, ksize, strides, + paddings, pool_process); } else if (pooling_type == "ave") { paddle::operators::math::Pool2dForwardFunctor< Place, paddle::operators::math::pool::avePool, T> pool2d_forward; paddle::operators::math::pool::avePool pool_process; - pool2d_forward(*input, *output, ksize, strides, paddings, - pool_process, (context.device_context())); + pool2d_forward(context.device_context(), *in_X, *out, ksize, strides, + paddings, pool_process); } } break; case 3: { @@ -67,15 +67,15 @@ class PoolKernel : public framework::OpKernel { Place, paddle::operators::math::pool::maxPool, T> pool3d_forward; paddle::operators::math::pool::maxPool pool_process; - pool3d_forward(*input, *output, ksize, strides, paddings, - pool_process, context.device_context()); + pool3d_forward(context.device_context(), *in_X, *out, ksize, strides, + paddings, pool_process); } else if (pooling_type == "ave") { paddle::operators::math::Pool3dForwardFunctor< Place, paddle::operators::math::pool::avePool, T> pool3d_forward; paddle::operators::math::pool::avePool pool_process; - pool3d_forward(*input, *output, ksize, strides, paddings, - pool_process, context.device_context()); + pool3d_forward(context.device_context(), *in_X, *out, ksize, strides, + paddings, pool_process); } } break; } @@ -86,26 +86,26 @@ template class PoolGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { - const Tensor* input = context.Input("Input"); - const Tensor* output = context.Input("Output"); - const Tensor* output_grad = - context.Input(framework::GradVarName("Output")); - Tensor* input_grad = - context.Output(framework::GradVarName("Input")); - - int global_pooling = context.Attr("global_pooling"); - std::string pooling_type = context.Attr("pooling_type"); + const Tensor* in_X = context.Input("X"); + const Tensor* out = context.Input("Out"); + const Tensor* out_grad = + context.Input(framework::GradVarName("Out")); + Tensor* in_X_grad = + context.Output(framework::GradVarName("X")); + + int global_pooling = context.Attr("globalPooling"); + std::string pooling_type = context.Attr("poolingType"); std::vector ksize = context.Attr>("ksize"); std::vector strides = context.Attr>("strides"); std::vector paddings = context.Attr>("paddings"); if (global_pooling == 1) { - for (size_t i = 0; i < ksize.size(); ++i) ksize[i] = input->dims()[i + 2]; + for (size_t i = 0; i < ksize.size(); ++i) ksize[i] = in_X->dims()[i + 2]; } - if (input_grad) { - input_grad->mutable_data(context.GetPlace()); - auto temp = framework::EigenVector::Flatten(*input_grad); + if (in_X_grad) { + in_X_grad->mutable_data(context.GetPlace()); + auto temp = framework::EigenVector::Flatten(*in_X_grad); temp.device(context.GetEigenDevice()) = temp.constant(static_cast(0)); @@ -116,17 +116,15 @@ class PoolGradKernel : public framework::OpKernel { Place, paddle::operators::math::pool::maxPool, T> pool2d_backward; paddle::operators::math::pool::maxPool pool_process; - pool2d_backward(*input, *input_grad, *output, *output_grad, ksize, - strides, paddings, pool_process, - context.device_context()); + pool2d_backward(context.device_context(), *in_X, *in_X_grad, *out, + *out_grad, ksize, strides, paddings, pool_process); } else if (pooling_type == "ave") { paddle::operators::math::Pool2dBackwardFunctor< Place, paddle::operators::math::pool::avePool, T> pool2d_backward; paddle::operators::math::pool::avePool pool_process; - pool2d_backward(*input, *input_grad, *output, *output_grad, ksize, - strides, paddings, pool_process, - context.device_context()); + pool2d_backward(context.device_context(), *in_X, *in_X_grad, *out, + *out_grad, ksize, strides, paddings, pool_process); } } break; case 3: { @@ -135,17 +133,15 @@ class PoolGradKernel : public framework::OpKernel { Place, paddle::operators::math::pool::maxPool, T> pool3d_backward; paddle::operators::math::pool::maxPool pool_process; - pool3d_backward(*input, *input_grad, *output, *output_grad, ksize, - strides, paddings, pool_process, - context.device_context()); + pool3d_backward(context.device_context(), *in_X, *in_X_grad, *out, + *out_grad, ksize, strides, paddings, pool_process); } else if (pooling_type == "ave") { paddle::operators::math::Pool3dBackwardFunctor< Place, paddle::operators::math::pool::avePool, T> pool3d_backward; paddle::operators::math::pool::avePool pool_process; - pool3d_backward(*input, *input_grad, *output, *output_grad, ksize, - strides, paddings, pool_process, - context.device_context()); + pool3d_backward(context.device_context(), *in_X, *in_X_grad, *out, + *out_grad, ksize, strides, paddings, pool_process); } } break; } diff --git a/python/paddle/v2/framework/tests/test_pool2d_op.py b/python/paddle/v2/framework/tests/test_pool2d_op.py index cf327508f7..2a8fedc037 100644 --- a/python/paddle/v2/framework/tests/test_pool2d_op.py +++ b/python/paddle/v2/framework/tests/test_pool2d_op.py @@ -47,23 +47,23 @@ class TestPool2d_Op(OpTest): input = np.random.random(self.shape).astype("float32") output = self.pool2D_forward_naive(input, self.ksize, self.strides, self.paddings) - self.inputs = {'Input': input} + self.inputs = {'X': input} self.attrs = { 'strides': self.strides, 'paddings': self.paddings, 'ksize': self.ksize, - 'pooling_type': self.pool_type, + 'poolingType': self.pool_type, } - self.outputs = {'Output': output} + self.outputs = {'Out': output} def test_check_output(self): self.check_output() def test_check_grad(self): if self.pool_type != "max": - self.check_grad(set(['Input']), 'Output', max_relative_error=0.07) + self.check_grad(set(['X']), 'Out', max_relative_error=0.07) def initTestCase(self): self.pool_type = "ave" diff --git a/python/paddle/v2/framework/tests/test_pool3d_op.py b/python/paddle/v2/framework/tests/test_pool3d_op.py index cfd0ced150..907ee0c0fe 100644 --- a/python/paddle/v2/framework/tests/test_pool3d_op.py +++ b/python/paddle/v2/framework/tests/test_pool3d_op.py @@ -57,23 +57,23 @@ class TestPool3d_Op(OpTest): input = np.random.random(self.shape).astype("float32") output = self.pool3D_forward_naive(input, self.ksize, self.strides, self.paddings) - self.inputs = {'Input': input} + self.inputs = {'X': input} self.attrs = { 'strides': self.strides, 'paddings': self.paddings, 'ksize': self.ksize, - 'pooling_type': self.pool_type, + 'poolingType': self.pool_type, } - self.outputs = {'Output': output} + self.outputs = {'Out': output} def test_check_output(self): self.check_output() def test_check_grad(self): if self.pool_type != "max": - self.check_grad(set(['Input']), 'Output', max_relative_error=0.07) + self.check_grad(set(['X']), 'Out', max_relative_error=0.07) def initTestCase(self): self.pool_type = "ave" From 6f61b5df7dd55f3f4ee5b7f42c7e31e9f84ea1be Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Fri, 22 Sep 2017 10:49:59 +0800 Subject: [PATCH 020/170] fix unit test --- paddle/operators/pool_op.cc | 13 +++--- .../v2/framework/tests/test_pool2d_op.py | 40 +++++++++++++++++-- .../v2/framework/tests/test_pool3d_op.py | 40 ++++++++++++++++--- 3 files changed, 77 insertions(+), 16 deletions(-) diff --git a/paddle/operators/pool_op.cc b/paddle/operators/pool_op.cc index c23adf63bf..bcda072a2d 100644 --- a/paddle/operators/pool_op.cc +++ b/paddle/operators/pool_op.cc @@ -46,20 +46,19 @@ class PoolOp : public framework::OperatorWithKernel { PADDLE_ENFORCE(pooling_type == "max" || pooling_type == "ave", "pooling_type should be 'max' or 'ave'"); - PADDLE_ENFORCE(ksize.size() == 2 || ksize.size() == 3, - "Pooling ksize should be 2-D or 3-D"); + PADDLE_ENFORCE(in_X->dims().size() == 4 || in_X->dims().size() == 5, + "Pooling intput should be 4-D or 5-D"); if (global_pooling == 1) { - for (size_t i = 0; i < ksize.size(); ++i) ksize[i] = in_X->dims()[i + 2]; + ksize.resize(static_cast(in_X->dims().size()) - 2); + for (size_t i = 0; i < ksize.size(); ++i) + ksize[i] = static_cast(in_X->dims()[i + 2]); } + if (ksize.size() == 2) { - PADDLE_ENFORCE_EQ(in_X->dims().size(), 4, - "Pool2DOp intput should be 4-D."); PADDLE_ENFORCE_EQ(strides.size(), 2, "Pool2DOp strides should be 2-D."); PADDLE_ENFORCE_EQ(paddings.size(), 2, "Pool2DOp paddings should be 2-D."); } else { - PADDLE_ENFORCE_EQ(in_X->dims().size(), 5, - "Pool3DOp intput should be 5-D."); PADDLE_ENFORCE_EQ(strides.size(), 3, "Pool3DOp strides should be 3-D."); PADDLE_ENFORCE_EQ(paddings.size(), 3, "Pool3DOp paddings should be 3-D."); } diff --git a/python/paddle/v2/framework/tests/test_pool2d_op.py b/python/paddle/v2/framework/tests/test_pool2d_op.py index 2a8fedc037..0efddc2881 100644 --- a/python/paddle/v2/framework/tests/test_pool2d_op.py +++ b/python/paddle/v2/framework/tests/test_pool2d_op.py @@ -3,9 +3,11 @@ import numpy as np from op_test import OpTest -def max_pool2D_forward_naive(x, ksize, strides, paddings=[0, 0]): +def max_pool2D_forward_naive(x, ksize, strides, paddings=[0, 0], global_pool=0): N, C, H, W = x.shape + if global_pool == 1: + ksize = [H, W] H_out = (H - ksize[0] + 2 * paddings[0]) / strides[0] + 1 W_out = (W - ksize[1] + 2 * paddings[1]) / strides[1] + 1 out = np.zeros((N, C, H_out, W_out)) @@ -21,9 +23,11 @@ def max_pool2D_forward_naive(x, ksize, strides, paddings=[0, 0]): return out -def ave_pool2D_forward_naive(x, ksize, strides, paddings=[0, 0]): +def ave_pool2D_forward_naive(x, ksize, strides, paddings=[0, 0], global_pool=0): N, C, H, W = x.shape + if global_pool == 1: + ksize = [H, W] H_out = (H - ksize[0] + 2 * paddings[0]) / strides[0] + 1 W_out = (W - ksize[1] + 2 * paddings[1]) / strides[1] + 1 out = np.zeros((N, C, H_out, W_out)) @@ -46,7 +50,7 @@ class TestPool2d_Op(OpTest): self.op_type = "pool2d" input = np.random.random(self.shape).astype("float32") output = self.pool2D_forward_naive(input, self.ksize, self.strides, - self.paddings) + self.paddings, self.global_pool) self.inputs = {'X': input} self.attrs = { @@ -54,6 +58,7 @@ class TestPool2d_Op(OpTest): 'paddings': self.paddings, 'ksize': self.ksize, 'poolingType': self.pool_type, + 'globalPooling': self.global_pool, } self.outputs = {'Out': output} @@ -66,6 +71,7 @@ class TestPool2d_Op(OpTest): self.check_grad(set(['X']), 'Out', max_relative_error=0.07) def initTestCase(self): + self.global_pool = 0 self.pool_type = "ave" self.pool2D_forward_naive = ave_pool2D_forward_naive self.shape = [2, 3, 5, 5] @@ -74,8 +80,21 @@ class TestPool2d_Op(OpTest): self.paddings = [0, 0] +class TestCase1(TestPool2d_Op): + def initTestCase(self): + self.global_pool = 0 + self.op_type = "pool2d" + self.pool_type = "ave" + self.pool2D_forward_naive = ave_pool2D_forward_naive + self.shape = [2, 3, 5, 5] + self.ksize = [3, 3] + self.strides = [1, 1] + self.paddings = [1, 1] + + class TestCase2(TestPool2d_Op): def initTestCase(self): + self.global_pool = 1 self.op_type = "pool2d" self.pool_type = "ave" self.pool2D_forward_naive = ave_pool2D_forward_naive @@ -85,8 +104,21 @@ class TestCase2(TestPool2d_Op): self.paddings = [1, 1] -class TestCase1(TestPool2d_Op): +class TestCase3(TestPool2d_Op): + def initTestCase(self): + self.global_pool = 0 + self.op_type = "pool2d" + self.pool_type = "max" + self.pool2D_forward_naive = max_pool2D_forward_naive + self.shape = [2, 3, 5, 5] + self.ksize = [3, 3] + self.strides = [1, 1] + self.paddings = [1, 1] + + +class TestCase4(TestPool2d_Op): def initTestCase(self): + self.global_pool = 1 self.op_type = "pool2d" self.pool_type = "max" self.pool2D_forward_naive = max_pool2D_forward_naive diff --git a/python/paddle/v2/framework/tests/test_pool3d_op.py b/python/paddle/v2/framework/tests/test_pool3d_op.py index 907ee0c0fe..4ba3d754d5 100644 --- a/python/paddle/v2/framework/tests/test_pool3d_op.py +++ b/python/paddle/v2/framework/tests/test_pool3d_op.py @@ -3,9 +3,11 @@ import numpy as np from op_test import OpTest -def max_pool3D_forward_naive(x, ksize, strides, paddings=[0, 0]): +def max_pool3D_forward_naive(x, ksize, strides, paddings=[0, 0], global_pool=0): N, C, D, H, W = x.shape + if global_pool == 1: + ksize = [D, H, W] D_out = (D - ksize[0] + 2 * paddings[0]) / strides[0] + 1 H_out = (H - ksize[1] + 2 * paddings[1]) / strides[1] + 1 W_out = (W - ksize[2] + 2 * paddings[2]) / strides[2] + 1 @@ -19,16 +21,17 @@ def max_pool3D_forward_naive(x, ksize, strides, paddings=[0, 0]): for j in xrange(W_out): w_start = np.max((j * strides[1] - paddings[1], 0)) w_end = np.min((j * strides[1] + ksize[1] - paddings[1], W)) - x_masked = x[:, :, d_start:d_end, h_start:h_end, w_start:w_end] out[:, :, k, i, j] = np.max(x_masked, axis=(2, 3, 4)) return out -def ave_pool3D_forward_naive(x, ksize, strides, paddings=[0, 0]): +def ave_pool3D_forward_naive(x, ksize, strides, paddings=[0, 0], global_pool=0): N, C, D, H, W = x.shape + if global_pool == 1: + ksize = [D, H, W] D_out = (D - ksize[0] + 2 * paddings[0]) / strides[0] + 1 H_out = (H - ksize[1] + 2 * paddings[1]) / strides[1] + 1 W_out = (W - ksize[2] + 2 * paddings[2]) / strides[2] + 1 @@ -42,7 +45,6 @@ def ave_pool3D_forward_naive(x, ksize, strides, paddings=[0, 0]): for j in xrange(W_out): w_start = np.max((j * strides[1] - paddings[1], 0)) w_end = np.min((j * strides[1] + ksize[1] - paddings[1], W)) - x_masked = x[:, :, d_start:d_end, h_start:h_end, w_start:w_end] out[:, :, k, i, j] = np.sum(x_masked, axis=(2, 3, 4)) / ( @@ -56,7 +58,7 @@ class TestPool3d_Op(OpTest): self.op_type = "pool3d" input = np.random.random(self.shape).astype("float32") output = self.pool3D_forward_naive(input, self.ksize, self.strides, - self.paddings) + self.paddings, self.global_pool) self.inputs = {'X': input} self.attrs = { @@ -64,6 +66,7 @@ class TestPool3d_Op(OpTest): 'paddings': self.paddings, 'ksize': self.ksize, 'poolingType': self.pool_type, + 'globalPooling': self.global_pool, } self.outputs = {'Out': output} @@ -76,6 +79,7 @@ class TestPool3d_Op(OpTest): self.check_grad(set(['X']), 'Out', max_relative_error=0.07) def initTestCase(self): + self.global_pool = 0 self.pool_type = "ave" self.pool3D_forward_naive = ave_pool3D_forward_naive self.shape = [2, 3, 5, 5, 5] @@ -86,6 +90,7 @@ class TestPool3d_Op(OpTest): class TestCase1(TestPool3d_Op): def initTestCase(self): + self.global_pool = 0 self.op_type = "pool3d" self.pool_type = "ave" self.pool3D_forward_naive = ave_pool3D_forward_naive @@ -97,6 +102,31 @@ class TestCase1(TestPool3d_Op): class TestCase2(TestPool3d_Op): def initTestCase(self): + self.global_pool = 1 + self.op_type = "pool3d" + self.pool_type = "ave" + self.pool3D_forward_naive = ave_pool3D_forward_naive + self.shape = [2, 3, 7, 7, 7] + self.ksize = [3, 3, 3] + self.strides = [1, 1, 1] + self.paddings = [1, 1, 1] + + +class TestCase3(TestPool3d_Op): + def initTestCase(self): + self.global_pool = 0 + self.op_type = "pool3d" + self.pool_type = "max" + self.pool3D_forward_naive = max_pool3D_forward_naive + self.shape = [2, 3, 5, 5, 5] + self.ksize = [3, 3, 3] + self.strides = [1, 1, 1] + self.paddings = [1, 1, 1] + + +class TestCase4(TestPool3d_Op): + def initTestCase(self): + self.global_pool = 1 self.op_type = "pool3d" self.pool_type = "max" self.pool3D_forward_naive = max_pool3D_forward_naive From 0417e4e4bf205566ca882f3878c0a64417f5da96 Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Fri, 22 Sep 2017 12:02:41 +0800 Subject: [PATCH 021/170] fix framework::LoDTensor => Tensor --- paddle/operators/math/pooling.h | 26 +++++++++++--------------- paddle/operators/pool_op.cc | 4 ++-- paddle/operators/pool_op.h | 3 +-- paddle/platform/hostdevice.h | 2 ++ 4 files changed, 16 insertions(+), 19 deletions(-) diff --git a/paddle/operators/math/pooling.h b/paddle/operators/math/pooling.h index 3f01c9dacb..627ece2ca4 100644 --- a/paddle/operators/math/pooling.h +++ b/paddle/operators/math/pooling.h @@ -16,17 +16,13 @@ limitations under the License. */ #include "paddle/framework/eigen.h" #include "paddle/framework/tensor.h" #include "paddle/platform/device_context.h" +#include "paddle/platform/hostdevice.h" namespace paddle { namespace operators { namespace math { ////////////////////// -#ifdef __NVCC__ -#define HL_DEVICE __device__ -#else -#define HL_DEVICE -#endif #define FLT_MAX __FLT_MAX__ ///////////////////// @@ -34,11 +30,11 @@ namespace pool { template class maxPool { public: - HL_DEVICE inline T initial() { return -(T)(FLT_MAX); } - HL_DEVICE inline void process(T& y, const T& x) { y = y > x ? y : x; } - HL_DEVICE inline void finalize(T& y, const T& poo_size) {} - HL_DEVICE inline void gradProcess(const T& x, const T& y, const T& dy, T& dx, - T scale) { + DEVICE inline T initial() { return static_cast(-FLT_MAX); } + DEVICE inline void process(T& y, const T& x) { y = y > x ? y : x; } + DEVICE inline void finalize(T& y, const T& poo_size) {} + DEVICE inline void gradProcess(const T& x, const T& y, const T& dy, T& dx, + T scale) { dx += dy * (x == y); } }; @@ -46,11 +42,11 @@ class maxPool { template class avePool { public: - HL_DEVICE inline T initial() { return 0; } - HL_DEVICE inline void process(T& y, const T& x) { y += x; } - HL_DEVICE inline void finalize(T& y, const T& poo_size) { y /= poo_size; } - HL_DEVICE inline void gradProcess(const T& x, const T& y, const T& dy, T& dx, - T scale) { + DEVICE inline T initial() { return static_cast(0); } + DEVICE inline void process(T& y, const T& x) { y += x; } + DEVICE inline void finalize(T& y, const T& poo_size) { y /= poo_size; } + DEVICE inline void gradProcess(const T& x, const T& y, const T& dy, T& dx, + T scale) { dx += (scale * dy); } }; diff --git a/paddle/operators/pool_op.cc b/paddle/operators/pool_op.cc index bcda072a2d..434aae811c 100644 --- a/paddle/operators/pool_op.cc +++ b/paddle/operators/pool_op.cc @@ -37,7 +37,7 @@ class PoolOp : public framework::OperatorWithKernel { // PADDLE_ENFORCE_NOT_NULL(Attr>("ksize"), "ksize should // not be null."); auto in_X = ctx.Input("X"); - auto out = ctx.Output("Out"); + auto out = ctx.Output("Out"); int global_pooling = Attr("globalPooling"); std::string pooling_type = Attr("poolingType"); std::vector ksize = Attr>("ksize"); @@ -78,7 +78,7 @@ class PoolOpGrad : public framework::OperatorWithKernel { protected: void InferShape(const framework::InferShapeContext &ctx) const override { auto in = ctx.Input("X"); - auto d_in = ctx.Output(framework::GradVarName("X")); + auto d_in = ctx.Output(framework::GradVarName("X")); if (d_in) d_in->Resize(in->dims()); } }; diff --git a/paddle/operators/pool_op.h b/paddle/operators/pool_op.h index 2e737f0a4b..5a40f76172 100644 --- a/paddle/operators/pool_op.h +++ b/paddle/operators/pool_op.h @@ -90,8 +90,7 @@ class PoolGradKernel : public framework::OpKernel { const Tensor* out = context.Input("Out"); const Tensor* out_grad = context.Input(framework::GradVarName("Out")); - Tensor* in_X_grad = - context.Output(framework::GradVarName("X")); + Tensor* in_X_grad = context.Output(framework::GradVarName("X")); int global_pooling = context.Attr("globalPooling"); std::string pooling_type = context.Attr("poolingType"); diff --git a/paddle/platform/hostdevice.h b/paddle/platform/hostdevice.h index e7de86b7b2..eb2df291cc 100644 --- a/paddle/platform/hostdevice.h +++ b/paddle/platform/hostdevice.h @@ -2,8 +2,10 @@ #ifdef __CUDACC__ #define HOSTDEVICE __host__ __device__ +#define DEVICE __device__ #define HOST __host__ #else #define HOSTDEVICE +#define DEVICE #define HOST #endif From f2ccc11f592203aebe73fd71fb5b4f538a22d128 Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Fri, 22 Sep 2017 18:15:46 +0800 Subject: [PATCH 022/170] fix pool doc (pool_op.cc) --- paddle/operators/pool_op.cc | 115 +++++++++++++++++++----------------- 1 file changed, 62 insertions(+), 53 deletions(-) diff --git a/paddle/operators/pool_op.cc b/paddle/operators/pool_op.cc index 434aae811c..c57922373f 100644 --- a/paddle/operators/pool_op.cc +++ b/paddle/operators/pool_op.cc @@ -32,10 +32,7 @@ class PoolOp : public framework::OperatorWithKernel { "X(Input) of Pooling should not be null."); PADDLE_ENFORCE_NOT_NULL(ctx.OutputVar("Out"), "Out(Output) of Pooling should not be null."); - // PADDLE_ENFORCE_NOT_NULL(Attr("poolingType"), - // "pooling_type should not be null."); - // PADDLE_ENFORCE_NOT_NULL(Attr>("ksize"), "ksize should - // not be null."); + auto in_X = ctx.Input("X"); auto out = ctx.Output("Out"); int global_pooling = Attr("globalPooling"); @@ -56,11 +53,15 @@ class PoolOp : public framework::OperatorWithKernel { } if (ksize.size() == 2) { - PADDLE_ENFORCE_EQ(strides.size(), 2, "Pool2DOp strides should be 2-D."); - PADDLE_ENFORCE_EQ(paddings.size(), 2, "Pool2DOp paddings should be 2-D."); + PADDLE_ENFORCE_EQ(strides.size(), 2, + "Pool2DOp strides size should be 2 elements."); + PADDLE_ENFORCE_EQ(paddings.size(), 2, + "Pool2DOp paddings size should be 2 elements"); } else { - PADDLE_ENFORCE_EQ(strides.size(), 3, "Pool3DOp strides should be 3-D."); - PADDLE_ENFORCE_EQ(paddings.size(), 3, "Pool3DOp paddings should be 3-D."); + PADDLE_ENFORCE_EQ(strides.size(), 3, + "Pool3DOp strides should be 3 elements."); + PADDLE_ENFORCE_EQ(paddings.size(), 3, + "Pool3DOp paddings should be 3 elements."); } std::vector output_shape({in_X->dims()[0], in_X->dims()[1]}); for (size_t i = 0; i < ksize.size(); ++i) { @@ -83,76 +84,84 @@ class PoolOpGrad : public framework::OperatorWithKernel { } }; -class Pool3dOpMaker : public framework::OpProtoAndCheckerMaker { +class Pool2dOpMaker : public framework::OpProtoAndCheckerMaker { public: - Pool3dOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + Pool2dOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { AddInput( "X", "The input tensor of pooling operator. " - "The format of input tensor is NCDHW. Where N is batch size, C is the " - "number of channels, D, H and W is the depth, height and width of " - "image."); + "The format of input tensor is NCHW. Where N is batch size, C is the " + "number of channels, H and W is the height and width of image."); AddOutput("Out", "The output tensor of pooling operator." - "The format of output tensor is also NCDHW."); + "The format of output tensor is also NCHW."); AddAttr("poolingType", - "poolingType of pooling operator.['max' or 'ave']"); + "poolingType of pooling operator." + "str constant equal to 'max' or 'ave'"); AddAttr>( - "ksize", "pooling size(depth, height, width) of pooling operator."); - AddAttr("globalPooling", - "default 0" - "whether to use the globalPooling.") + "ksize", "pooling size(height, width) of pooling operator."); + AddAttr( + "globalPooling", + "whether to use the globalPooling." + "int constant equal to 0 or 1" + "default 0" + "If globalPooling = 1, ksize is ignored and need not be specified.") .SetDefault(0); - AddAttr>( - "strides", - "default {1,1,1}" - "strides(depth, height, width) of pooling operator.") - .SetDefault({1, 1, 1}); - AddAttr>( - "paddings", - "default {0,0,0}" - "paddings(depth, height, width) of pooling operator.") - .SetDefault({0, 0, 0}); + AddAttr>("strides", + "strides(height, width) of pooling operator." + "default {1,1}") + .SetDefault({1, 1}); + AddAttr>("paddings", + "paddings(height, width) of pooling operator." + "default {0,0}") + .SetDefault({0, 0}); + AddComment(R"DOC( -The pooling3d operation calculates the output based on +The pooling2d operation calculates the output based on the input, poolingType and ksize, strides, paddings parameters. )DOC"); } }; - -class Pool2dOpMaker : public framework::OpProtoAndCheckerMaker { +class Pool3dOpMaker : public framework::OpProtoAndCheckerMaker { public: - Pool2dOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) + Pool3dOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput( - "X", - "The input tensor of pooling operator. " - "The format of input tensor is NCHW. Where N is batch size, C is the " - "number of channels, H and W is the height and width of image."); + AddInput("X", + "The input tensor of pooling operator. " + "The format of input tensor is NCDHW. Where N is batch size, C is " + "the " + "number of channels, D, H and W is the depth, height and width of " + "image."); AddOutput("Out", "The output tensor of pooling operator." - "The format of output tensor is also NCHW."); + "The format of output tensor is also NCDHW."); AddAttr("poolingType", - "poolingType of pooling operator.['max' or 'ave']"); + "poolingType of pooling operator." + "str constant equal to 'max' or 'ave'"); AddAttr>( - "ksize", "pooling size(height, width) of pooling operator."); - AddAttr("globalPooling", - "default 0" - "whether to use the globalPooling.[0 or 1]") + "ksize", "pooling size(depth, height, width) of pooling operator."); + AddAttr( + "globalPooling", + "whether to use the globalPooling." + "int constant equal to 0 or 1" + "default 0" + "If globalPooling = 1, ksize is ignored and need not be specified.") .SetDefault(0); - AddAttr>("strides", - "default {1, 1}" - "strides(height, width) of pooling operator.") - .SetDefault({1, 1}); - AddAttr>("paddings", - "default {0, 0}" - "paddings(height, width) of pooling operator.") - .SetDefault({0, 0}); + AddAttr>( + "strides", + "strides(depth, height, width) of pooling operator." + "default {1,1,1}") + .SetDefault({1, 1, 1}); + AddAttr>( + "paddings", + "paddings(depth, height, width) of pooling operator." + "default {0,0,0}") + .SetDefault({0, 0, 0}); AddComment(R"DOC( -The pooling2d operation calculates the output based on +The pooling3d operation calculates the output based on the input, poolingType and ksize, strides, paddings parameters. )DOC"); } From b0a47b149d981fd3a4ad8d9bf03c9bd0d0cc96d1 Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Sat, 23 Sep 2017 14:55:02 +0800 Subject: [PATCH 023/170] remove CHECK in *.cu --- paddle/operators/math/pooling.cu | 9 --------- 1 file changed, 9 deletions(-) diff --git a/paddle/operators/math/pooling.cu b/paddle/operators/math/pooling.cu index 124006942c..09e6bd9000 100644 --- a/paddle/operators/math/pooling.cu +++ b/paddle/operators/math/pooling.cu @@ -140,8 +140,6 @@ class Pool2dForwardFunctor { output_width, ksize_height, ksize_width, stride_height, stride_width, padding_height, padding_width, pool_process); - - // CHECK_SYNC("Pool2dForwardKernel failed"); } }; @@ -186,8 +184,6 @@ class Pool2dBackwardFunctor { input_channels, input_height, input_width, output_height, output_width, ksize_height, ksize_width, stride_height, stride_width, padding_height, padding_width, pool_process); - - // CHECK_SYNC("KernelPool2dBackward failed"); } }; @@ -247,7 +243,6 @@ __global__ void KernelPool3DForward( } int pool_size = (dend - dstart) * (hend - hstart) * (wend - wstart); pool_process.finalize(ele, static_cast(pool_size)); - output_data[index] = ele; } } @@ -361,8 +356,6 @@ class Pool3dForwardFunctor { ksize_depth, ksize_height, ksize_width, stride_depth, stride_height, stride_width, padding_depth, padding_height, padding_width, pool_process); - - // CHECK_SYNC("Pool2dForwardKernel failed"); } }; @@ -415,8 +408,6 @@ class Pool3dBackwardFunctor { output_height, output_width, ksize_depth, ksize_height, ksize_width, stride_depth, stride_height, stride_width, padding_depth, padding_height, padding_width, pool_process); - - // CHECK_SYNC("KernelPool2dBackward failed"); } }; From 9b1431b8d932741ce9f976a860aabaefd176397e Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Fri, 22 Sep 2017 23:27:18 +0800 Subject: [PATCH 024/170] fix maxpool backward functor and add unit test --- paddle/operators/math/CMakeLists.txt | 2 + paddle/operators/math/pool_test_maxPool2d.cc | 150 ++++++++++++++ paddle/operators/math/pool_test_maxPool3d.cc | 154 +++++++++++++++ paddle/operators/math/pooling.cc | 142 ++++++++++++++ paddle/operators/math/pooling.cu | 196 +++++++++++++++++++ paddle/operators/math/pooling.h | 20 ++ 6 files changed, 664 insertions(+) create mode 100644 paddle/operators/math/pool_test_maxPool2d.cc create mode 100644 paddle/operators/math/pool_test_maxPool3d.cc diff --git a/paddle/operators/math/CMakeLists.txt b/paddle/operators/math/CMakeLists.txt index 185708cdaa..1233a9ea32 100644 --- a/paddle/operators/math/CMakeLists.txt +++ b/paddle/operators/math/CMakeLists.txt @@ -7,3 +7,5 @@ endif() nv_test(math_function_test SRCS math_function_test.cc DEPS math_function tensor) cc_test(im2col_test SRCS im2col_test.cc DEPS math_function tensor) +cc_test(pool_test_maxPool2d_test SRCS pool_test_maxPool2d.cc DEPS math_function tensor) +cc_test(pool_test_maxPool3d_test SRCS pool_test_maxPool3d.cc DEPS math_function tensor) diff --git a/paddle/operators/math/pool_test_maxPool2d.cc b/paddle/operators/math/pool_test_maxPool2d.cc new file mode 100644 index 0000000000..8ddf1d79c7 --- /dev/null +++ b/paddle/operators/math/pool_test_maxPool2d.cc @@ -0,0 +1,150 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ + +#include +#include "paddle/operators/math/pooling.h" + +#include "paddle/memory/memcpy.h" +#include "paddle/platform/enforce.h" + +#include +#include + +#ifndef PADDLE_ONLY_CPU + +template +void testPool2d(paddle::platform::DeviceContext& context, PooType pool_process, + paddle::framework::Tensor& input, + paddle::framework::Tensor& input_grad, + paddle::framework::Tensor& output, + paddle::framework::Tensor& output_grad, std::vector& ksize, + std::vector& strides, std::vector& paddings) { + paddle::operators::math::Pool2dForwardFunctor + pool2d_forward; + pool2d_forward(context, input, output, ksize, strides, paddings, + pool_process); + + int times = 50; + clock_t start, finish; + double totaltime; + + // Pool2dBackwardFunctor + start = clock(); + for (int i = 0; i < times; ++i) { + paddle::operators::math::Pool2dBackwardFunctor + pool2d_backward; + pool2d_backward(context, input, input_grad, output, output_grad, ksize, + strides, paddings, pool_process); + PADDLE_ENFORCE(cudaStreamSynchronize(0), + "cudaStreamSynchronize failed in pool2d_backward CopyFrom"); + } + finish = clock(); + totaltime = (double)(finish - start) / CLOCKS_PER_SEC; + totaltime /= times; + std::cout << "\nPool3dBackwardFunctor: " << totaltime << "s" << std::endl; + + // MaxPool3dBackwardFunctor + start = clock(); + for (int j = 0; j < times; ++j) { + paddle::operators::math::MaxPool2dBackwardFunctor< + paddle::platform::GPUPlace, float> + maxpool2d_backward; + maxpool2d_backward(context, input, input_grad, output, output_grad, ksize, + strides, paddings); + PADDLE_ENFORCE( + cudaStreamSynchronize(0), + "cudaStreamSynchronize failed in maxpool2d_backward CopyFrom"); + } + finish = clock(); + totaltime = (double)(finish - start) / CLOCKS_PER_SEC; + totaltime /= times; + std::cout << "\nMaxPool3dBackwardFunctor: " << totaltime << "s" << std::endl; +} + +void test2dPool() { + using paddle::platform::DeviceContext; + using paddle::platform::CUDADeviceContext; + using paddle::platform::GPUPlace; + + paddle::framework::Tensor input_tmp; + paddle::framework::Tensor output_tmp; + paddle::framework::Tensor input; + paddle::framework::Tensor input_grad; + paddle::framework::Tensor output; + paddle::framework::Tensor output_grad; + + int batch = 32; + int channel = 32; + int input_height = 128; + int input_width = 128; + int in_len = batch * channel * input_height * input_width; + std::vector ksize({3, 3}); + std::vector strides({1, 1}); + std::vector paddings({0, 0}); + + int output_height = + (input_height - ksize[0] + 2 * paddings[0]) / strides[0] + 1; + int output_width = + (input_width - ksize[1] + 2 * paddings[1]) / strides[1] + 1; + int output_len = output_height * output_width; + + input_tmp.mutable_data({batch, channel, input_height, input_width}, + paddle::platform::CPUPlace()); + output_tmp.mutable_data({batch, channel, output_height, output_width}, + paddle::platform::CPUPlace()); + + float* arr = new float[in_len]; + auto* place = new paddle::platform::GPUPlace(); + + float* input_ptr = input_tmp.data(); + for (int i = 0; i < in_len; ++i) arr[i] = i; // rand() / double(RAND_MAX/2); + memcpy(input_ptr, arr, in_len * sizeof(float)); + input.CopyFrom(input_tmp, *place); + + input_ptr = input_tmp.data(); + for (int i = 0; i < in_len; ++i) arr[i] = 0; + memcpy(input_ptr, arr, in_len * sizeof(float)); + input_grad.CopyFrom(input_tmp, *place); + + // output + input_ptr = output_tmp.data(); + for (int i = 0; i < output_len; ++i) + arr[i] = 0; // rand() / double(RAND_MAX/2); + memcpy(input_ptr, arr, output_len * sizeof(float)); + output.CopyFrom(input_tmp, *place); + + // output + input_ptr = output_tmp.data(); + for (int i = 0; i < output_len; ++i) + arr[i] = 1; // rand() / double(RAND_MAX/2); + memcpy(input_ptr, arr, output_len * sizeof(float)); + output_grad.CopyFrom(input_tmp, *place); + + paddle::platform::DeviceContext* context = + new paddle::platform::CUDADeviceContext(paddle::platform::GPUPlace()); + paddle::operators::math::pool::maxPool pool_process; + + testPool2d>( + *context, pool_process, input, input_grad, output, output_grad, ksize, + strides, paddings); +} + +int main() { + // testPool3d(); + test2dPool(); + // testPool3d(); +} +#endif \ No newline at end of file diff --git a/paddle/operators/math/pool_test_maxPool3d.cc b/paddle/operators/math/pool_test_maxPool3d.cc new file mode 100644 index 0000000000..000b006a12 --- /dev/null +++ b/paddle/operators/math/pool_test_maxPool3d.cc @@ -0,0 +1,154 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ + +#include +#include "paddle/operators/math/pooling.h" + +#include "paddle/memory/memcpy.h" +#include "paddle/platform/enforce.h" + +#include +#include + +#ifndef PADDLE_ONLY_CPU + +template +void testPool3d(paddle::platform::DeviceContext& context, PooType pool_process, + paddle::framework::Tensor& input, + paddle::framework::Tensor& input_grad, + paddle::framework::Tensor& output, + paddle::framework::Tensor& output_grad, std::vector& ksize, + std::vector& strides, std::vector& paddings) { + paddle::operators::math::Pool3dForwardFunctor + pool3d_forward; + pool3d_forward(context, input, output, ksize, strides, paddings, + pool_process); + + int times = 50; + clock_t start, finish; + double totaltime; + + // Pool3dBackwardFunctor + start = clock(); + for (int i = 0; i < times; ++i) { + paddle::operators::math::Pool3dBackwardFunctor + pool3d_backward; + pool3d_backward(context, input, input_grad, output, output_grad, ksize, + strides, paddings, pool_process); + PADDLE_ENFORCE(cudaStreamSynchronize(0), + "cudaStreamSynchronize failed in pool3d_backward CopyFrom"); + } + finish = clock(); + totaltime = (double)(finish - start) / CLOCKS_PER_SEC; + totaltime /= times; + std::cout << "\nPool3dBackwardFunctor: " << totaltime << "s" << std::endl; + + // MaxPool3dBackwardFunctor + start = clock(); + for (int j = 0; j < times; ++j) { + paddle::operators::math::MaxPool3dBackwardFunctor< + paddle::platform::GPUPlace, float> + maxpool3d_backward; + maxpool3d_backward(context, input, input_grad, output, output_grad, ksize, + strides, paddings); + PADDLE_ENFORCE( + cudaStreamSynchronize(0), + "cudaStreamSynchronize failed in maxpool3d_backward CopyFrom"); + } + finish = clock(); + totaltime = (double)(finish - start) / CLOCKS_PER_SEC; + totaltime /= times; + std::cout << "\nMaxPool3dBackwardFunctor: " << totaltime << "s" << std::endl; +} + +void test3dPool() { + using paddle::platform::DeviceContext; + using paddle::platform::CUDADeviceContext; + using paddle::platform::GPUPlace; + + paddle::framework::Tensor input_tmp; + paddle::framework::Tensor output_tmp; + paddle::framework::Tensor input; + paddle::framework::Tensor input_grad; + paddle::framework::Tensor output; + paddle::framework::Tensor output_grad; + + int batch = 32; + int channel = 4; + int input_depth = 4; + int input_height = 128; + int input_width = 128; + int in_len = batch * channel * input_depth * input_height * input_width; + std::vector ksize({3, 3, 3}); + std::vector strides({2, 2, 2}); + std::vector paddings({1, 1, 1}); + + int output_depth = + (input_depth - ksize[0] + 2 * paddings[0]) / strides[0] + 1; + int output_height = + (input_height - ksize[1] + 2 * paddings[1]) / strides[1] + 1; + int output_width = + (input_width - ksize[2] + 2 * paddings[2]) / strides[2] + 1; + + int output_len = output_depth * output_height * output_width; + + input_tmp.mutable_data( + {batch, channel, input_depth, input_height, input_width}, + paddle::platform::CPUPlace()); + output_tmp.mutable_data( + {batch, channel, output_depth, output_height, output_width}, + paddle::platform::CPUPlace()); + + float* arr = new float[in_len]; + auto* place = new paddle::platform::GPUPlace(); + + // input + float* input_ptr = input_tmp.data(); + for (int i = 0; i < in_len; ++i) arr[i] = i; // rand() / double(RAND_MAX/2); + memcpy(input_ptr, arr, in_len * sizeof(float)); + input.CopyFrom(input_tmp, *place); + + // input_grad + input_ptr = input_tmp.data(); + for (int i = 0; i < in_len; ++i) arr[i] = 0; + memcpy(input_ptr, arr, in_len * sizeof(float)); + input_grad.CopyFrom(input_tmp, *place); + + // output + input_ptr = output_tmp.data(); + for (int i = 0; i < output_len; ++i) + arr[i] = 0; // rand() / double(RAND_MAX/2); + memcpy(input_ptr, arr, output_len * sizeof(float)); + output.CopyFrom(input_tmp, *place); + + // output_grad + input_ptr = output_tmp.data(); + for (int i = 0; i < output_len; ++i) + arr[i] = 1; // rand() / double(RAND_MAX/2); + memcpy(input_ptr, arr, output_len * sizeof(float)); + output_grad.CopyFrom(input_tmp, *place); + + paddle::platform::DeviceContext* context = + new paddle::platform::CUDADeviceContext(paddle::platform::GPUPlace()); + paddle::operators::math::pool::maxPool pool_process; + + testPool3d>( + *context, pool_process, input, input_grad, output, output_grad, ksize, + strides, paddings); +} + +int main() { test3dPool(); } +#endif \ No newline at end of file diff --git a/paddle/operators/math/pooling.cc b/paddle/operators/math/pooling.cc index 5ce748ff08..7c616023ca 100644 --- a/paddle/operators/math/pooling.cc +++ b/paddle/operators/math/pooling.cc @@ -134,6 +134,70 @@ class Pool2dBackwardFunctor { } }; +template +class MaxPool2dBackwardFunctor { + public: + void operator()(const platform::DeviceContext& context, + const framework::Tensor& input, framework::Tensor& input_grad, + const framework::Tensor& output, + const framework::Tensor& output_grad, std::vector& ksize, + std::vector& strides, std::vector& paddings) { + const int batch_size = input.dims()[0]; + const int input_height = input.dims()[2]; + const int input_width = input.dims()[3]; + const int output_channels = output.dims()[1]; + const int output_height = output.dims()[2]; + const int output_width = output.dims()[3]; + const int ksize_height = ksize[0]; + const int ksize_width = ksize[1]; + const int stride_height = strides[0]; + const int stride_width = strides[1]; + const int padding_height = paddings[0]; + const int padding_width = paddings[1]; + const int input_stride = input_height * input_width; + const int output_stride = output_height * output_width; + + const T* input_data = input.data(); + const T* output_data = output.data(); + const T* output_grad_data = output_grad.data(); + T* input_grad_data = input_grad.mutable_data(context.GetPlace()); + + for (int i = 0; i < batch_size; i++) { + for (int c = 0; c < output_channels; ++c) { + for (int ph = 0; ph < output_height; ++ph) { + int hstart = ph * stride_height - padding_height; + int hend = std::min(hstart + ksize_height, input_height); + hstart = std::max(hstart, 0); + for (int pw = 0; pw < output_width; ++pw) { + int wstart = pw * stride_width - padding_width; + int wend = std::min(wstart + ksize_width, input_width); + wstart = std::max(wstart, 0); + + bool stop = false; + for (int h = hstart; h < hend && !stop; ++h) { + for (int w = wstart; w < wend && !stop; ++w) { + int input_idx = h * input_width + w; + int output_idx = ph * output_width + pw; + if (input_data[input_idx] == output_data[output_idx]) { + input_grad_data[input_idx] += output_grad_data[output_idx]; + stop = true; + } + } + } + } + } + input_data += input_stride; + output_data += output_stride; + input_grad_data += input_stride; + output_grad_data += output_stride; + } + } + } +}; + +template class MaxPool2dBackwardFunctor; +template class MaxPool2dBackwardFunctor; + template class Pool2dForwardFunctor< platform::CPUPlace, paddle::operators::math::pool::maxPool, float>; template class Pool2dForwardFunctor< @@ -299,6 +363,84 @@ class Pool3dBackwardFunctor { } }; +template +class MaxPool3dBackwardFunctor { + public: + void operator()(const platform::DeviceContext& context, + const framework::Tensor& input, framework::Tensor& input_grad, + const framework::Tensor& output, + const framework::Tensor& output_grad, std::vector& ksize, + std::vector& strides, std::vector& paddings) { + const int batch_size = input.dims()[0]; + const int input_depth = input.dims()[2]; + const int input_height = input.dims()[3]; + const int input_width = input.dims()[4]; + const int output_channels = output.dims()[1]; + const int output_depth = output.dims()[2]; + const int output_height = output.dims()[3]; + const int output_width = output.dims()[4]; + const int ksize_depth = ksize[0]; + const int ksize_height = ksize[1]; + const int ksize_width = ksize[2]; + const int stride_depth = strides[0]; + const int stride_height = strides[1]; + const int stride_width = strides[2]; + const int padding_depth = paddings[0]; + const int padding_height = paddings[1]; + const int padding_width = paddings[2]; + const int input_stride = input_depth * input_height * input_width; + const int output_stride = output_depth * output_height * output_width; + + const T* input_data = input.data(); + const T* output_data = output.data(); + const T* output_grad_data = output_grad.data(); + T* input_grad_data = input_grad.mutable_data(context.GetPlace()); + + for (int i = 0; i < batch_size; i++) { + for (int c = 0; c < output_channels; ++c) { + for (int pd = 0; pd < output_depth; ++pd) { + int dstart = pd * stride_depth - padding_depth; + int dend = std::min(dstart + ksize_depth, input_depth); + dstart = std::max(dstart, 0); + for (int ph = 0; ph < output_height; ++ph) { + int hstart = ph * stride_height - padding_height; + int hend = std::min(hstart + ksize_height, input_height); + hstart = std::max(hstart, 0); + for (int pw = 0; pw < output_width; ++pw) { + int wstart = pw * stride_width - padding_width; + int wend = std::min(wstart + ksize_width, input_width); + wstart = std::max(wstart, 0); + bool stop = false; + for (int d = dstart; d < dend && !stop; ++d) { + for (int h = hstart; h < hend && !stop; ++h) { + for (int w = wstart; w < wend && !stop; ++w) { + int input_idx = (d * input_height + h) * input_width + w; + int output_idx = + (pd * output_height + ph) * output_width + pw; + + if (input_data[input_idx] == output_data[output_idx]) { + input_grad_data[input_idx] += + output_grad_data[output_idx]; + stop = true; + } + } + } + } + } + } + } + input_data += input_stride; + output_data += output_stride; + input_grad_data += input_stride; + output_grad_data += output_stride; + } + } + } +}; + +template class MaxPool3dBackwardFunctor; +template class MaxPool3dBackwardFunctor; + template class Pool3dForwardFunctor< platform::CPUPlace, paddle::operators::math::pool::maxPool, float>; template class Pool3dForwardFunctor< diff --git a/paddle/operators/math/pooling.cu b/paddle/operators/math/pooling.cu index 09e6bd9000..347270abc3 100644 --- a/paddle/operators/math/pooling.cu +++ b/paddle/operators/math/pooling.cu @@ -102,6 +102,51 @@ __global__ void KernelPool2dBackward( } } +template +__global__ void KernelMaxPool2dBackward( + const int nthreads, const T* input_data, const T* output_data, + const T* output_grad, T* input_grad, const int channels, + const int input_height, const int input_width, const int output_height, + const int output_width, const int ksize_height, const int ksize_width, + const int stride_height, const int stride_width, const int padding_height, + const int padding_width) { + int index = blockIdx.x * blockDim.x + threadIdx.x; + if (index < nthreads) { + int pw = index % output_width; + int ph = (index / output_width) % output_height; + int c = (index / output_width / output_height) % channels; + int batch_idx = index / output_width / output_height / channels; + + int hstart = ph * stride_height - padding_height; + int hend = min(hstart + ksize_height, input_height); + hstart = max(hstart, 0); + + int wstart = pw * stride_width - padding_width; + int wend = min(wstart + ksize_width, input_width); + wstart = max(wstart, 0); + + input_data += (batch_idx * channels + c) * input_height * input_width; + input_grad += (batch_idx * channels + c) * input_height * input_width; + + T ele = output_data[index]; + int maxIndex = -1; + bool stop = false; + for (int h = hstart; h < hend && !stop; ++h) { + for (int w = wstart; w < wend && !stop; ++w) { + if (ele == input_data[h * input_width + w]) { + maxIndex = h * input_width + w; + stop = true; + } + } + } + + if (maxIndex != -1) { + // atomic add + atomicAdd(input_grad + maxIndex, output_grad[index]); + } + } +} + template class Pool2dForwardFunctor { public: @@ -187,6 +232,52 @@ class Pool2dBackwardFunctor { } }; +template +class MaxPool2dBackwardFunctor { + public: + void operator()(const platform::DeviceContext& context, + const framework::Tensor& input, framework::Tensor& input_grad, + const framework::Tensor& output, + const framework::Tensor& output_grad, std::vector& ksize, + std::vector& strides, std::vector& paddings) { + const int batch_size = input.dims()[0]; + const int input_channels = input.dims()[1]; + const int input_height = input.dims()[2]; + const int input_width = input.dims()[3]; + const int output_channels = output.dims()[1]; + const int output_height = output.dims()[2]; + const int output_width = output.dims()[3]; + const int ksize_height = ksize[0]; + const int ksize_width = ksize[1]; + const int stride_height = strides[0]; + const int stride_width = strides[1]; + const int padding_height = paddings[0]; + const int padding_width = paddings[1]; + + const T* input_data = input.data(); + const T* output_data = output.data(); + const T* output_grad_data = output_grad.data(); + T* input_grad_data = input_grad.mutable_data(context.GetPlace()); + + int nthreads = batch_size * output_channels * output_height * output_width; + int blocks = (nthreads + 1024 - 1) / 1024; + dim3 threads(1024, 1); + dim3 grid(blocks, 1); + + KernelMaxPool2dBackward< + T><<(context) + .stream()>>>( + nthreads, input_data, output_data, output_grad_data, input_grad_data, + input_channels, input_height, input_width, output_height, output_width, + ksize_height, ksize_width, stride_height, stride_width, padding_height, + padding_width); + } +}; + +template class MaxPool2dBackwardFunctor; +// template class MaxPool2dBackwardFunctor; + template class Pool2dForwardFunctor< platform::GPUPlace, paddle::operators::math::pool::maxPool, float>; template class Pool2dForwardFunctor< @@ -311,6 +402,58 @@ __global__ void KernelPool3DBackward( } } +template +__global__ void KernelMaxPool3DBackward( + const int nthreads, const T* input_data, const T* output_data, + const T* output_grad, T* input_grad, const int channels, + const int input_depth, const int input_height, const int input_width, + const int output_depth, const int output_height, const int output_width, + const int ksize_depth, const int ksize_height, const int ksize_width, + const int stride_depth, const int stride_height, const int stride_width, + const int padding_depth, const int padding_height, + const int padding_width) { + for (int index = blockIdx.x * blockDim.x + threadIdx.x; index < (nthreads); + index += blockDim.x * gridDim.x) { + int pw = index % output_width; + int ph = (index / output_width) % output_height; + int pd = (index / output_width / output_height) % output_depth; + int c = (index / output_width / output_height / output_depth) % channels; + int batch_idx = + index / output_width / output_height / output_depth / channels; + int dstart = pd * stride_depth - padding_depth; + int hstart = ph * stride_height - padding_height; + int wstart = pw * stride_width - padding_width; + int dend = min(dstart + ksize_depth, input_depth); + int hend = min(hstart + ksize_height, input_height); + int wend = min(wstart + ksize_width, input_width); + dstart = max(dstart, 0); + hstart = max(hstart, 0); + wstart = max(wstart, 0); + T ele = output_data[index]; + bool stop = false; + int maxIdx = -1; + input_data += + (batch_idx * channels + c) * input_depth * input_height * input_width; + input_grad += + (batch_idx * channels + c) * input_depth * input_height * input_width; + + for (int d = dstart; d < dend && !stop; ++d) { + for (int h = hstart; h < hend && !stop; ++h) { + for (int w = wstart; w < wend && !stop; ++w) { + if (ele == input_data[(d * input_height + h) * input_width + w]) { + stop = true; + maxIdx = (d * input_height + h) * input_width + w; + } + } + } + } + if (maxIdx != -1) { + // atomic add + atomicAdd(input_grad + maxIdx, output_grad[index]); + } + } +} + template class Pool3dForwardFunctor { public: @@ -411,6 +554,59 @@ class Pool3dBackwardFunctor { } }; +template +class MaxPool3dBackwardFunctor { + public: + void operator()(const platform::DeviceContext& context, + const framework::Tensor& input, framework::Tensor& input_grad, + const framework::Tensor& output, + const framework::Tensor& output_grad, std::vector& ksize, + std::vector& strides, std::vector& paddings) { + const int batch_size = input.dims()[0]; + const int input_channels = input.dims()[1]; + const int input_depth = input.dims()[2]; + const int input_height = input.dims()[3]; + const int input_width = input.dims()[4]; + const int output_channels = output.dims()[1]; + const int output_depth = output.dims()[2]; + const int output_height = output.dims()[3]; + const int output_width = output.dims()[4]; + const int ksize_depth = ksize[0]; + const int ksize_height = ksize[1]; + const int ksize_width = ksize[2]; + const int stride_depth = strides[0]; + const int stride_height = strides[1]; + const int stride_width = strides[2]; + const int padding_depth = paddings[0]; + const int padding_height = paddings[1]; + const int padding_width = paddings[2]; + + const T* input_data = input.data(); + const T* output_data = output.data(); + const T* output_grad_data = output_grad.data(); + T* input_grad_data = input_grad.mutable_data(context.GetPlace()); + + int nthreads = batch_size * output_channels * output_depth * output_height * + output_width; + int blocks = (nthreads + 1024 - 1) / 1024; + dim3 threads(1024, 1); + dim3 grid(blocks, 1); + + KernelMaxPool3DBackward< + T><<(context) + .stream()>>>( + nthreads, input_data, output_data, output_grad_data, input_grad_data, + input_channels, input_depth, input_height, input_width, output_depth, + output_height, output_width, ksize_depth, ksize_height, ksize_width, + stride_depth, stride_height, stride_width, padding_depth, + padding_height, padding_width); + } +}; + +template class MaxPool3dBackwardFunctor; +// template class MaxPool3dBackwardFunctor; + template class Pool3dForwardFunctor< platform::GPUPlace, paddle::operators::math::pool::maxPool, float>; template class Pool3dForwardFunctor< diff --git a/paddle/operators/math/pooling.h b/paddle/operators/math/pooling.h index 627ece2ca4..4b8b02d374 100644 --- a/paddle/operators/math/pooling.h +++ b/paddle/operators/math/pooling.h @@ -72,6 +72,16 @@ class Pool2dBackwardFunctor { PoolProcess pool_process); }; +template +class MaxPool2dBackwardFunctor { + public: + void operator()(const platform::DeviceContext& context, + const framework::Tensor& input, framework::Tensor& input_grad, + const framework::Tensor& output, + const framework::Tensor& output_grad, std::vector& ksize, + std::vector& strides, std::vector& paddings); +}; + template class Pool3dForwardFunctor { public: @@ -92,6 +102,16 @@ class Pool3dBackwardFunctor { PoolProcess pool_process); }; +template +class MaxPool3dBackwardFunctor { + public: + void operator()(const platform::DeviceContext& context, + const framework::Tensor& input, framework::Tensor& input_grad, + const framework::Tensor& output, + const framework::Tensor& output_grad, std::vector& ksize, + std::vector& strides, std::vector& paddings); +}; + } // namespace math } // namespace operators } // namespace paddle From b72854389e3d7e24f24a82f2608ee470cd9b43ff Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Mon, 25 Sep 2017 20:14:12 +0800 Subject: [PATCH 025/170] Fix (According to the review) --- paddle/operators/math/pooling.cc | 32 ++++++++++++------- paddle/operators/math/pooling.cu | 32 ++++++++++++------- paddle/operators/math/pooling.h | 17 +++++++--- paddle/operators/pool_op.cc | 8 ++--- paddle/operators/pool_op.h | 32 +++++++++---------- .../v2/framework/tests/test_pool2d_op.py | 14 ++++---- .../v2/framework/tests/test_pool3d_op.py | 14 ++++---- 7 files changed, 87 insertions(+), 62 deletions(-) diff --git a/paddle/operators/math/pooling.cc b/paddle/operators/math/pooling.cc index 5ce748ff08..72b260e729 100644 --- a/paddle/operators/math/pooling.cc +++ b/paddle/operators/math/pooling.cc @@ -137,19 +137,23 @@ class Pool2dBackwardFunctor { template class Pool2dForwardFunctor< platform::CPUPlace, paddle::operators::math::pool::maxPool, float>; template class Pool2dForwardFunctor< - platform::CPUPlace, paddle::operators::math::pool::avePool, float>; + platform::CPUPlace, paddle::operators::math::pool::avgPool, float>; template class Pool2dBackwardFunctor< - platform::CPUPlace, paddle::operators::math::pool::maxPool, float>; + platform::CPUPlace, paddle::operators::math::pool::maxPoolGrad, + float>; template class Pool2dBackwardFunctor< - platform::CPUPlace, paddle::operators::math::pool::avePool, float>; + platform::CPUPlace, paddle::operators::math::pool::avgPoolGrad, + float>; template class Pool2dForwardFunctor< platform::CPUPlace, paddle::operators::math::pool::maxPool, double>; template class Pool2dForwardFunctor< - platform::CPUPlace, paddle::operators::math::pool::avePool, double>; + platform::CPUPlace, paddle::operators::math::pool::avgPool, double>; template class Pool2dBackwardFunctor< - platform::CPUPlace, paddle::operators::math::pool::maxPool, double>; + platform::CPUPlace, paddle::operators::math::pool::maxPoolGrad, + double>; template class Pool2dBackwardFunctor< - platform::CPUPlace, paddle::operators::math::pool::avePool, double>; + platform::CPUPlace, paddle::operators::math::pool::avgPoolGrad, + double>; template class Pool3dForwardFunctor { @@ -302,19 +306,23 @@ class Pool3dBackwardFunctor { template class Pool3dForwardFunctor< platform::CPUPlace, paddle::operators::math::pool::maxPool, float>; template class Pool3dForwardFunctor< - platform::CPUPlace, paddle::operators::math::pool::avePool, float>; + platform::CPUPlace, paddle::operators::math::pool::avgPool, float>; template class Pool3dBackwardFunctor< - platform::CPUPlace, paddle::operators::math::pool::maxPool, float>; + platform::CPUPlace, paddle::operators::math::pool::maxPoolGrad, + float>; template class Pool3dBackwardFunctor< - platform::CPUPlace, paddle::operators::math::pool::avePool, float>; + platform::CPUPlace, paddle::operators::math::pool::avgPoolGrad, + float>; template class Pool3dForwardFunctor< platform::CPUPlace, paddle::operators::math::pool::maxPool, double>; template class Pool3dForwardFunctor< - platform::CPUPlace, paddle::operators::math::pool::avePool, double>; + platform::CPUPlace, paddle::operators::math::pool::avgPool, double>; template class Pool3dBackwardFunctor< - platform::CPUPlace, paddle::operators::math::pool::maxPool, double>; + platform::CPUPlace, paddle::operators::math::pool::maxPoolGrad, + double>; template class Pool3dBackwardFunctor< - platform::CPUPlace, paddle::operators::math::pool::avePool, double>; + platform::CPUPlace, paddle::operators::math::pool::avgPoolGrad, + double>; } // namespace math } // namespace operators } // namespace paddle diff --git a/paddle/operators/math/pooling.cu b/paddle/operators/math/pooling.cu index 09e6bd9000..619fd64b97 100644 --- a/paddle/operators/math/pooling.cu +++ b/paddle/operators/math/pooling.cu @@ -190,19 +190,23 @@ class Pool2dBackwardFunctor { template class Pool2dForwardFunctor< platform::GPUPlace, paddle::operators::math::pool::maxPool, float>; template class Pool2dForwardFunctor< - platform::GPUPlace, paddle::operators::math::pool::avePool, float>; + platform::GPUPlace, paddle::operators::math::pool::avgPool, float>; template class Pool2dBackwardFunctor< - platform::GPUPlace, paddle::operators::math::pool::maxPool, float>; + platform::GPUPlace, paddle::operators::math::pool::maxPoolGrad, + float>; template class Pool2dBackwardFunctor< - platform::GPUPlace, paddle::operators::math::pool::avePool, float>; + platform::GPUPlace, paddle::operators::math::pool::avgPoolGrad, + float>; template class Pool2dForwardFunctor< platform::GPUPlace, paddle::operators::math::pool::maxPool, double>; template class Pool2dForwardFunctor< - platform::GPUPlace, paddle::operators::math::pool::avePool, double>; + platform::GPUPlace, paddle::operators::math::pool::avgPool, double>; template class Pool2dBackwardFunctor< - platform::GPUPlace, paddle::operators::math::pool::maxPool, double>; + platform::GPUPlace, paddle::operators::math::pool::maxPoolGrad, + double>; template class Pool2dBackwardFunctor< - platform::GPUPlace, paddle::operators::math::pool::avePool, double>; + platform::GPUPlace, paddle::operators::math::pool::avgPoolGrad, + double>; template __global__ void KernelPool3DForward( @@ -414,19 +418,23 @@ class Pool3dBackwardFunctor { template class Pool3dForwardFunctor< platform::GPUPlace, paddle::operators::math::pool::maxPool, float>; template class Pool3dForwardFunctor< - platform::GPUPlace, paddle::operators::math::pool::avePool, float>; + platform::GPUPlace, paddle::operators::math::pool::avgPool, float>; template class Pool3dBackwardFunctor< - platform::GPUPlace, paddle::operators::math::pool::maxPool, float>; + platform::GPUPlace, paddle::operators::math::pool::maxPoolGrad, + float>; template class Pool3dBackwardFunctor< - platform::GPUPlace, paddle::operators::math::pool::avePool, float>; + platform::GPUPlace, paddle::operators::math::pool::avgPoolGrad, + float>; template class Pool3dForwardFunctor< platform::GPUPlace, paddle::operators::math::pool::maxPool, double>; template class Pool3dForwardFunctor< - platform::GPUPlace, paddle::operators::math::pool::avePool, double>; + platform::GPUPlace, paddle::operators::math::pool::avgPool, double>; template class Pool3dBackwardFunctor< - platform::GPUPlace, paddle::operators::math::pool::maxPool, double>; + platform::GPUPlace, paddle::operators::math::pool::maxPoolGrad, + double>; template class Pool3dBackwardFunctor< - platform::GPUPlace, paddle::operators::math::pool::avePool, double>; + platform::GPUPlace, paddle::operators::math::pool::avgPoolGrad, + double>; } // namespace math } // namespace operators diff --git a/paddle/operators/math/pooling.h b/paddle/operators/math/pooling.h index 627ece2ca4..b975ad31af 100644 --- a/paddle/operators/math/pooling.h +++ b/paddle/operators/math/pooling.h @@ -33,6 +33,18 @@ class maxPool { DEVICE inline T initial() { return static_cast(-FLT_MAX); } DEVICE inline void process(T& y, const T& x) { y = y > x ? y : x; } DEVICE inline void finalize(T& y, const T& poo_size) {} +}; + +template +class avgPool { + public: + DEVICE inline T initial() { return static_cast(0); } + DEVICE inline void process(T& y, const T& x) { y += x; } + DEVICE inline void finalize(T& y, const T& poo_size) { y /= poo_size; } +}; +template +class maxPoolGrad { + public: DEVICE inline void gradProcess(const T& x, const T& y, const T& dy, T& dx, T scale) { dx += dy * (x == y); @@ -40,11 +52,8 @@ class maxPool { }; template -class avePool { +class avgPoolGrad { public: - DEVICE inline T initial() { return static_cast(0); } - DEVICE inline void process(T& y, const T& x) { y += x; } - DEVICE inline void finalize(T& y, const T& poo_size) { y /= poo_size; } DEVICE inline void gradProcess(const T& x, const T& y, const T& dy, T& dx, T scale) { dx += (scale * dy); diff --git a/paddle/operators/pool_op.cc b/paddle/operators/pool_op.cc index c57922373f..e51249beb9 100644 --- a/paddle/operators/pool_op.cc +++ b/paddle/operators/pool_op.cc @@ -41,8 +41,8 @@ class PoolOp : public framework::OperatorWithKernel { std::vector strides = Attr>("strides"); std::vector paddings = Attr>("paddings"); - PADDLE_ENFORCE(pooling_type == "max" || pooling_type == "ave", - "pooling_type should be 'max' or 'ave'"); + PADDLE_ENFORCE(pooling_type == "max" || pooling_type == "avg", + "pooling_type should be 'max' or 'avg'"); PADDLE_ENFORCE(in_X->dims().size() == 4 || in_X->dims().size() == 5, "Pooling intput should be 4-D or 5-D"); @@ -99,7 +99,7 @@ class Pool2dOpMaker : public framework::OpProtoAndCheckerMaker { AddAttr("poolingType", "poolingType of pooling operator." - "str constant equal to 'max' or 'ave'"); + "str constant equal to 'max' or 'avg'"); AddAttr>( "ksize", "pooling size(height, width) of pooling operator."); AddAttr( @@ -140,7 +140,7 @@ class Pool3dOpMaker : public framework::OpProtoAndCheckerMaker { AddAttr("poolingType", "poolingType of pooling operator." - "str constant equal to 'max' or 'ave'"); + "str constant equal to 'max' or 'avg'"); AddAttr>( "ksize", "pooling size(depth, height, width) of pooling operator."); AddAttr( diff --git a/paddle/operators/pool_op.h b/paddle/operators/pool_op.h index 5a40f76172..6f1d05272c 100644 --- a/paddle/operators/pool_op.h +++ b/paddle/operators/pool_op.h @@ -52,11 +52,11 @@ class PoolKernel : public framework::OpKernel { pool2d_forward(context.device_context(), *in_X, *out, ksize, strides, paddings, pool_process); - } else if (pooling_type == "ave") { + } else if (pooling_type == "avg") { paddle::operators::math::Pool2dForwardFunctor< - Place, paddle::operators::math::pool::avePool, T> + Place, paddle::operators::math::pool::avgPool, T> pool2d_forward; - paddle::operators::math::pool::avePool pool_process; + paddle::operators::math::pool::avgPool pool_process; pool2d_forward(context.device_context(), *in_X, *out, ksize, strides, paddings, pool_process); } @@ -69,11 +69,11 @@ class PoolKernel : public framework::OpKernel { paddle::operators::math::pool::maxPool pool_process; pool3d_forward(context.device_context(), *in_X, *out, ksize, strides, paddings, pool_process); - } else if (pooling_type == "ave") { + } else if (pooling_type == "avg") { paddle::operators::math::Pool3dForwardFunctor< - Place, paddle::operators::math::pool::avePool, T> + Place, paddle::operators::math::pool::avgPool, T> pool3d_forward; - paddle::operators::math::pool::avePool pool_process; + paddle::operators::math::pool::avgPool pool_process; pool3d_forward(context.device_context(), *in_X, *out, ksize, strides, paddings, pool_process); } @@ -112,16 +112,16 @@ class PoolGradKernel : public framework::OpKernel { case 2: { if (pooling_type == "max") { paddle::operators::math::Pool2dBackwardFunctor< - Place, paddle::operators::math::pool::maxPool, T> + Place, paddle::operators::math::pool::maxPoolGrad, T> pool2d_backward; - paddle::operators::math::pool::maxPool pool_process; + paddle::operators::math::pool::maxPoolGrad pool_process; pool2d_backward(context.device_context(), *in_X, *in_X_grad, *out, *out_grad, ksize, strides, paddings, pool_process); - } else if (pooling_type == "ave") { + } else if (pooling_type == "avg") { paddle::operators::math::Pool2dBackwardFunctor< - Place, paddle::operators::math::pool::avePool, T> + Place, paddle::operators::math::pool::avgPoolGrad, T> pool2d_backward; - paddle::operators::math::pool::avePool pool_process; + paddle::operators::math::pool::avgPoolGrad pool_process; pool2d_backward(context.device_context(), *in_X, *in_X_grad, *out, *out_grad, ksize, strides, paddings, pool_process); } @@ -129,16 +129,16 @@ class PoolGradKernel : public framework::OpKernel { case 3: { if (pooling_type == "max") { paddle::operators::math::Pool3dBackwardFunctor< - Place, paddle::operators::math::pool::maxPool, T> + Place, paddle::operators::math::pool::maxPoolGrad, T> pool3d_backward; - paddle::operators::math::pool::maxPool pool_process; + paddle::operators::math::pool::maxPoolGrad pool_process; pool3d_backward(context.device_context(), *in_X, *in_X_grad, *out, *out_grad, ksize, strides, paddings, pool_process); - } else if (pooling_type == "ave") { + } else if (pooling_type == "avg") { paddle::operators::math::Pool3dBackwardFunctor< - Place, paddle::operators::math::pool::avePool, T> + Place, paddle::operators::math::pool::avgPoolGrad, T> pool3d_backward; - paddle::operators::math::pool::avePool pool_process; + paddle::operators::math::pool::avgPoolGrad pool_process; pool3d_backward(context.device_context(), *in_X, *in_X_grad, *out, *out_grad, ksize, strides, paddings, pool_process); } diff --git a/python/paddle/v2/framework/tests/test_pool2d_op.py b/python/paddle/v2/framework/tests/test_pool2d_op.py index 0efddc2881..e8e5218073 100644 --- a/python/paddle/v2/framework/tests/test_pool2d_op.py +++ b/python/paddle/v2/framework/tests/test_pool2d_op.py @@ -23,7 +23,7 @@ def max_pool2D_forward_naive(x, ksize, strides, paddings=[0, 0], global_pool=0): return out -def ave_pool2D_forward_naive(x, ksize, strides, paddings=[0, 0], global_pool=0): +def avg_pool2D_forward_naive(x, ksize, strides, paddings=[0, 0], global_pool=0): N, C, H, W = x.shape if global_pool == 1: @@ -72,8 +72,8 @@ class TestPool2d_Op(OpTest): def initTestCase(self): self.global_pool = 0 - self.pool_type = "ave" - self.pool2D_forward_naive = ave_pool2D_forward_naive + self.pool_type = "avg" + self.pool2D_forward_naive = avg_pool2D_forward_naive self.shape = [2, 3, 5, 5] self.ksize = [3, 3] self.strides = [1, 1] @@ -84,8 +84,8 @@ class TestCase1(TestPool2d_Op): def initTestCase(self): self.global_pool = 0 self.op_type = "pool2d" - self.pool_type = "ave" - self.pool2D_forward_naive = ave_pool2D_forward_naive + self.pool_type = "avg" + self.pool2D_forward_naive = avg_pool2D_forward_naive self.shape = [2, 3, 5, 5] self.ksize = [3, 3] self.strides = [1, 1] @@ -96,8 +96,8 @@ class TestCase2(TestPool2d_Op): def initTestCase(self): self.global_pool = 1 self.op_type = "pool2d" - self.pool_type = "ave" - self.pool2D_forward_naive = ave_pool2D_forward_naive + self.pool_type = "avg" + self.pool2D_forward_naive = avg_pool2D_forward_naive self.shape = [2, 3, 5, 5] self.ksize = [3, 3] self.strides = [1, 1] diff --git a/python/paddle/v2/framework/tests/test_pool3d_op.py b/python/paddle/v2/framework/tests/test_pool3d_op.py index 4ba3d754d5..2a3de89a70 100644 --- a/python/paddle/v2/framework/tests/test_pool3d_op.py +++ b/python/paddle/v2/framework/tests/test_pool3d_op.py @@ -27,7 +27,7 @@ def max_pool3D_forward_naive(x, ksize, strides, paddings=[0, 0], global_pool=0): return out -def ave_pool3D_forward_naive(x, ksize, strides, paddings=[0, 0], global_pool=0): +def avg_pool3D_forward_naive(x, ksize, strides, paddings=[0, 0], global_pool=0): N, C, D, H, W = x.shape if global_pool == 1: @@ -80,8 +80,8 @@ class TestPool3d_Op(OpTest): def initTestCase(self): self.global_pool = 0 - self.pool_type = "ave" - self.pool3D_forward_naive = ave_pool3D_forward_naive + self.pool_type = "avg" + self.pool3D_forward_naive = avg_pool3D_forward_naive self.shape = [2, 3, 5, 5, 5] self.ksize = [3, 3, 3] self.strides = [1, 1, 1] @@ -92,8 +92,8 @@ class TestCase1(TestPool3d_Op): def initTestCase(self): self.global_pool = 0 self.op_type = "pool3d" - self.pool_type = "ave" - self.pool3D_forward_naive = ave_pool3D_forward_naive + self.pool_type = "avg" + self.pool3D_forward_naive = avg_pool3D_forward_naive self.shape = [2, 3, 7, 7, 7] self.ksize = [3, 3, 3] self.strides = [1, 1, 1] @@ -104,8 +104,8 @@ class TestCase2(TestPool3d_Op): def initTestCase(self): self.global_pool = 1 self.op_type = "pool3d" - self.pool_type = "ave" - self.pool3D_forward_naive = ave_pool3D_forward_naive + self.pool_type = "avg" + self.pool3D_forward_naive = avg_pool3D_forward_naive self.shape = [2, 3, 7, 7, 7] self.ksize = [3, 3, 3] self.strides = [1, 1, 1] From 8c478b36e2f1c723d48b5c0e96fd77b7d950c467 Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Mon, 25 Sep 2017 21:21:52 +0800 Subject: [PATCH 026/170] fix Atrr check --- paddle/operators/pool_op.cc | 81 +++++++++++++++++++++++++++++++++---- 1 file changed, 74 insertions(+), 7 deletions(-) diff --git a/paddle/operators/pool_op.cc b/paddle/operators/pool_op.cc index e51249beb9..9c1b7ea41d 100644 --- a/paddle/operators/pool_op.cc +++ b/paddle/operators/pool_op.cc @@ -101,7 +101,8 @@ class Pool2dOpMaker : public framework::OpProtoAndCheckerMaker { "poolingType of pooling operator." "str constant equal to 'max' or 'avg'"); AddAttr>( - "ksize", "pooling size(height, width) of pooling operator."); + "ksize", "pooling size(height, width) of pooling operator.") + .AddCustomChecker(GreaterThanChecker_pool({0, 0})); AddAttr( "globalPooling", "whether to use the globalPooling." @@ -112,17 +113,49 @@ class Pool2dOpMaker : public framework::OpProtoAndCheckerMaker { AddAttr>("strides", "strides(height, width) of pooling operator." "default {1,1}") - .SetDefault({1, 1}); + .SetDefault({1, 1}) + .AddCustomChecker(GreaterThanChecker_pool({0, 0})); AddAttr>("paddings", "paddings(height, width) of pooling operator." "default {0,0}") - .SetDefault({0, 0}); - + .SetDefault({0, 0}) + .AddCustomChecker(EqualGreaterThanChecker_pool({0, 0})); AddComment(R"DOC( The pooling2d operation calculates the output based on the input, poolingType and ksize, strides, paddings parameters. )DOC"); } + + private: + struct GreaterThanChecker_pool { + public: + explicit GreaterThanChecker_pool(std::vector lower_bound) + : lower_bound_(lower_bound) {} + void operator()(std::vector &value) const { + PADDLE_ENFORCE(value.size() == lower_bound_.size(), "equal check fails."); + for (size_t i = 0; i < value.size(); ++i) { + PADDLE_ENFORCE(value[i] > lower_bound_[i], "larger_than check fails."); + } + } + + private: + std::vector lower_bound_; + }; + + struct EqualGreaterThanChecker_pool { + public: + explicit EqualGreaterThanChecker_pool(std::vector lower_bound) + : lower_bound_(lower_bound) {} + void operator()(std::vector &value) const { + PADDLE_ENFORCE(value.size() == lower_bound_.size(), "equal check fails."); + for (size_t i = 0; i < value.size(); ++i) { + PADDLE_ENFORCE(value[i] >= lower_bound_[i], "larger_than check fails."); + } + } + + private: + std::vector lower_bound_; + }; }; class Pool3dOpMaker : public framework::OpProtoAndCheckerMaker { public: @@ -142,7 +175,8 @@ class Pool3dOpMaker : public framework::OpProtoAndCheckerMaker { "poolingType of pooling operator." "str constant equal to 'max' or 'avg'"); AddAttr>( - "ksize", "pooling size(depth, height, width) of pooling operator."); + "ksize", "pooling size(depth, height, width) of pooling operator.") + .AddCustomChecker(GreaterThanChecker_pool({0, 0, 0})); AddAttr( "globalPooling", "whether to use the globalPooling." @@ -154,17 +188,50 @@ class Pool3dOpMaker : public framework::OpProtoAndCheckerMaker { "strides", "strides(depth, height, width) of pooling operator." "default {1,1,1}") - .SetDefault({1, 1, 1}); + .SetDefault({1, 1, 1}) + .AddCustomChecker(GreaterThanChecker_pool({0, 0, 0})); AddAttr>( "paddings", "paddings(depth, height, width) of pooling operator." "default {0,0,0}") - .SetDefault({0, 0, 0}); + .SetDefault({0, 0, 0}) + .AddCustomChecker(EqualGreaterThanChecker_pool({0, 0, 0})); AddComment(R"DOC( The pooling3d operation calculates the output based on the input, poolingType and ksize, strides, paddings parameters. )DOC"); } + + private: + struct GreaterThanChecker_pool { + public: + explicit GreaterThanChecker_pool(std::vector lower_bound) + : lower_bound_(lower_bound) {} + void operator()(std::vector &value) const { + PADDLE_ENFORCE(value.size() == lower_bound_.size(), "equal check fails."); + for (size_t i = 0; i < value.size(); ++i) { + PADDLE_ENFORCE(value[i] > lower_bound_[i], "larger_than check fails."); + } + } + + private: + std::vector lower_bound_; + }; + + struct EqualGreaterThanChecker_pool { + public: + explicit EqualGreaterThanChecker_pool(std::vector lower_bound) + : lower_bound_(lower_bound) {} + void operator()(std::vector &value) const { + PADDLE_ENFORCE(value.size() == lower_bound_.size(), "equal check fails."); + for (size_t i = 0; i < value.size(); ++i) { + PADDLE_ENFORCE(value[i] >= lower_bound_[i], "larger_than check fails."); + } + } + + private: + std::vector lower_bound_; + }; }; } // namespace operators } // namespace paddle From f6e69d74124995fd0dac30b6d7d3717ef2a9241a Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Mon, 25 Sep 2017 23:10:03 +0800 Subject: [PATCH 027/170] fix maxpool backward functor --- paddle/operators/math/pool_test_maxPool2d.cc | 22 ++++++++++++-------- paddle/operators/math/pool_test_maxPool3d.cc | 21 +++++++++++-------- paddle/operators/math/pooling.cc | 4 ++-- paddle/operators/pool_op.h | 12 ++++------- 4 files changed, 31 insertions(+), 28 deletions(-) diff --git a/paddle/operators/math/pool_test_maxPool2d.cc b/paddle/operators/math/pool_test_maxPool2d.cc index 8ddf1d79c7..63b6b5657b 100644 --- a/paddle/operators/math/pool_test_maxPool2d.cc +++ b/paddle/operators/math/pool_test_maxPool2d.cc @@ -23,16 +23,17 @@ #ifndef PADDLE_ONLY_CPU -template -void testPool2d(paddle::platform::DeviceContext& context, PooType pool_process, - paddle::framework::Tensor& input, +template +void testPool2d(paddle::platform::DeviceContext& context, PoolType pool_process, + PoolGradType poolGrad_process, paddle::framework::Tensor& input, paddle::framework::Tensor& input_grad, paddle::framework::Tensor& output, paddle::framework::Tensor& output_grad, std::vector& ksize, std::vector& strides, std::vector& paddings) { paddle::operators::math::Pool2dForwardFunctor + PoolType, float> pool2d_forward; + pool2d_forward(context, input, output, ksize, strides, paddings, pool_process); @@ -44,10 +45,11 @@ void testPool2d(paddle::platform::DeviceContext& context, PooType pool_process, start = clock(); for (int i = 0; i < times; ++i) { paddle::operators::math::Pool2dBackwardFunctor + PoolGradType, float> pool2d_backward; pool2d_backward(context, input, input_grad, output, output_grad, ksize, - strides, paddings, pool_process); + strides, paddings, poolGrad_process); + PADDLE_ENFORCE(cudaStreamSynchronize(0), "cudaStreamSynchronize failed in pool2d_backward CopyFrom"); } @@ -136,10 +138,12 @@ void test2dPool() { paddle::platform::DeviceContext* context = new paddle::platform::CUDADeviceContext(paddle::platform::GPUPlace()); paddle::operators::math::pool::maxPool pool_process; + paddle::operators::math::pool::maxPoolGrad poolGrad_process; - testPool2d>( - *context, pool_process, input, input_grad, output, output_grad, ksize, - strides, paddings); + testPool2d, + paddle::operators::math::pool::maxPoolGrad>( + *context, pool_process, poolGrad_process, input, input_grad, output, + output_grad, ksize, strides, paddings); } int main() { diff --git a/paddle/operators/math/pool_test_maxPool3d.cc b/paddle/operators/math/pool_test_maxPool3d.cc index 000b006a12..ce572164c4 100644 --- a/paddle/operators/math/pool_test_maxPool3d.cc +++ b/paddle/operators/math/pool_test_maxPool3d.cc @@ -23,15 +23,15 @@ #ifndef PADDLE_ONLY_CPU -template -void testPool3d(paddle::platform::DeviceContext& context, PooType pool_process, - paddle::framework::Tensor& input, +template +void testPool3d(paddle::platform::DeviceContext& context, PoolType pool_process, + PoolGradType poolGrad_process, paddle::framework::Tensor& input, paddle::framework::Tensor& input_grad, paddle::framework::Tensor& output, paddle::framework::Tensor& output_grad, std::vector& ksize, std::vector& strides, std::vector& paddings) { paddle::operators::math::Pool3dForwardFunctor + PoolType, float> pool3d_forward; pool3d_forward(context, input, output, ksize, strides, paddings, pool_process); @@ -44,10 +44,10 @@ void testPool3d(paddle::platform::DeviceContext& context, PooType pool_process, start = clock(); for (int i = 0; i < times; ++i) { paddle::operators::math::Pool3dBackwardFunctor + PoolGradType, float> pool3d_backward; pool3d_backward(context, input, input_grad, output, output_grad, ksize, - strides, paddings, pool_process); + strides, paddings, poolGrad_process); PADDLE_ENFORCE(cudaStreamSynchronize(0), "cudaStreamSynchronize failed in pool3d_backward CopyFrom"); } @@ -145,9 +145,12 @@ void test3dPool() { new paddle::platform::CUDADeviceContext(paddle::platform::GPUPlace()); paddle::operators::math::pool::maxPool pool_process; - testPool3d>( - *context, pool_process, input, input_grad, output, output_grad, ksize, - strides, paddings); + paddle::operators::math::pool::maxPoolGrad poolGrad_process; + + testPool3d, + paddle::operators::math::pool::maxPoolGrad>( + *context, pool_process, poolGrad_process, input, input_grad, output, + output_grad, ksize, strides, paddings); } int main() { test3dPool(); } diff --git a/paddle/operators/math/pooling.cc b/paddle/operators/math/pooling.cc index 6539b80678..cb1f9d7285 100644 --- a/paddle/operators/math/pooling.cc +++ b/paddle/operators/math/pooling.cc @@ -196,7 +196,7 @@ class MaxPool2dBackwardFunctor { }; template class MaxPool2dBackwardFunctor; -template class MaxPool2dBackwardFunctor; +// template class MaxPool2dBackwardFunctor; template class Pool2dForwardFunctor< platform::CPUPlace, paddle::operators::math::pool::maxPool, float>; @@ -443,7 +443,7 @@ class MaxPool3dBackwardFunctor { }; template class MaxPool3dBackwardFunctor; -template class MaxPool3dBackwardFunctor; +// template class MaxPool3dBackwardFunctor; template class Pool3dForwardFunctor< platform::CPUPlace, paddle::operators::math::pool::maxPool, float>; diff --git a/paddle/operators/pool_op.h b/paddle/operators/pool_op.h index 6f1d05272c..763103d884 100644 --- a/paddle/operators/pool_op.h +++ b/paddle/operators/pool_op.h @@ -111,12 +111,10 @@ class PoolGradKernel : public framework::OpKernel { switch (ksize.size()) { case 2: { if (pooling_type == "max") { - paddle::operators::math::Pool2dBackwardFunctor< - Place, paddle::operators::math::pool::maxPoolGrad, T> + paddle::operators::math::MaxPool2dBackwardFunctor pool2d_backward; - paddle::operators::math::pool::maxPoolGrad pool_process; pool2d_backward(context.device_context(), *in_X, *in_X_grad, *out, - *out_grad, ksize, strides, paddings, pool_process); + *out_grad, ksize, strides, paddings); } else if (pooling_type == "avg") { paddle::operators::math::Pool2dBackwardFunctor< Place, paddle::operators::math::pool::avgPoolGrad, T> @@ -128,12 +126,10 @@ class PoolGradKernel : public framework::OpKernel { } break; case 3: { if (pooling_type == "max") { - paddle::operators::math::Pool3dBackwardFunctor< - Place, paddle::operators::math::pool::maxPoolGrad, T> + paddle::operators::math::MaxPool3dBackwardFunctor pool3d_backward; - paddle::operators::math::pool::maxPoolGrad pool_process; pool3d_backward(context.device_context(), *in_X, *in_X_grad, *out, - *out_grad, ksize, strides, paddings, pool_process); + *out_grad, ksize, strides, paddings); } else if (pooling_type == "avg") { paddle::operators::math::Pool3dBackwardFunctor< Place, paddle::operators::math::pool::avgPoolGrad, T> From dfc8d3c1c172362afe4cf8fd7ad317cec435ee13 Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Tue, 26 Sep 2017 14:25:35 +0800 Subject: [PATCH 028/170] Fix (According to the review) --- paddle/operators/math/CMakeLists.txt | 2 - paddle/operators/math/pool_test_maxPool2d.cc | 154 ------------------ paddle/operators/math/pool_test_maxPool3d.cc | 157 ------------------- paddle/operators/math/pooling.cc | 129 +++++++-------- paddle/operators/math/pooling.cu | 144 ++++++++--------- paddle/operators/math/pooling.h | 35 ++--- paddle/operators/pool_op.cc | 46 +++--- paddle/operators/pool_op.cu | 21 +-- paddle/operators/pool_op.h | 73 ++++----- 9 files changed, 216 insertions(+), 545 deletions(-) delete mode 100644 paddle/operators/math/pool_test_maxPool2d.cc delete mode 100644 paddle/operators/math/pool_test_maxPool3d.cc diff --git a/paddle/operators/math/CMakeLists.txt b/paddle/operators/math/CMakeLists.txt index 1233a9ea32..185708cdaa 100644 --- a/paddle/operators/math/CMakeLists.txt +++ b/paddle/operators/math/CMakeLists.txt @@ -7,5 +7,3 @@ endif() nv_test(math_function_test SRCS math_function_test.cc DEPS math_function tensor) cc_test(im2col_test SRCS im2col_test.cc DEPS math_function tensor) -cc_test(pool_test_maxPool2d_test SRCS pool_test_maxPool2d.cc DEPS math_function tensor) -cc_test(pool_test_maxPool3d_test SRCS pool_test_maxPool3d.cc DEPS math_function tensor) diff --git a/paddle/operators/math/pool_test_maxPool2d.cc b/paddle/operators/math/pool_test_maxPool2d.cc deleted file mode 100644 index 63b6b5657b..0000000000 --- a/paddle/operators/math/pool_test_maxPool2d.cc +++ /dev/null @@ -1,154 +0,0 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. */ - -#include -#include "paddle/operators/math/pooling.h" - -#include "paddle/memory/memcpy.h" -#include "paddle/platform/enforce.h" - -#include -#include - -#ifndef PADDLE_ONLY_CPU - -template -void testPool2d(paddle::platform::DeviceContext& context, PoolType pool_process, - PoolGradType poolGrad_process, paddle::framework::Tensor& input, - paddle::framework::Tensor& input_grad, - paddle::framework::Tensor& output, - paddle::framework::Tensor& output_grad, std::vector& ksize, - std::vector& strides, std::vector& paddings) { - paddle::operators::math::Pool2dForwardFunctor - pool2d_forward; - - pool2d_forward(context, input, output, ksize, strides, paddings, - pool_process); - - int times = 50; - clock_t start, finish; - double totaltime; - - // Pool2dBackwardFunctor - start = clock(); - for (int i = 0; i < times; ++i) { - paddle::operators::math::Pool2dBackwardFunctor - pool2d_backward; - pool2d_backward(context, input, input_grad, output, output_grad, ksize, - strides, paddings, poolGrad_process); - - PADDLE_ENFORCE(cudaStreamSynchronize(0), - "cudaStreamSynchronize failed in pool2d_backward CopyFrom"); - } - finish = clock(); - totaltime = (double)(finish - start) / CLOCKS_PER_SEC; - totaltime /= times; - std::cout << "\nPool3dBackwardFunctor: " << totaltime << "s" << std::endl; - - // MaxPool3dBackwardFunctor - start = clock(); - for (int j = 0; j < times; ++j) { - paddle::operators::math::MaxPool2dBackwardFunctor< - paddle::platform::GPUPlace, float> - maxpool2d_backward; - maxpool2d_backward(context, input, input_grad, output, output_grad, ksize, - strides, paddings); - PADDLE_ENFORCE( - cudaStreamSynchronize(0), - "cudaStreamSynchronize failed in maxpool2d_backward CopyFrom"); - } - finish = clock(); - totaltime = (double)(finish - start) / CLOCKS_PER_SEC; - totaltime /= times; - std::cout << "\nMaxPool3dBackwardFunctor: " << totaltime << "s" << std::endl; -} - -void test2dPool() { - using paddle::platform::DeviceContext; - using paddle::platform::CUDADeviceContext; - using paddle::platform::GPUPlace; - - paddle::framework::Tensor input_tmp; - paddle::framework::Tensor output_tmp; - paddle::framework::Tensor input; - paddle::framework::Tensor input_grad; - paddle::framework::Tensor output; - paddle::framework::Tensor output_grad; - - int batch = 32; - int channel = 32; - int input_height = 128; - int input_width = 128; - int in_len = batch * channel * input_height * input_width; - std::vector ksize({3, 3}); - std::vector strides({1, 1}); - std::vector paddings({0, 0}); - - int output_height = - (input_height - ksize[0] + 2 * paddings[0]) / strides[0] + 1; - int output_width = - (input_width - ksize[1] + 2 * paddings[1]) / strides[1] + 1; - int output_len = output_height * output_width; - - input_tmp.mutable_data({batch, channel, input_height, input_width}, - paddle::platform::CPUPlace()); - output_tmp.mutable_data({batch, channel, output_height, output_width}, - paddle::platform::CPUPlace()); - - float* arr = new float[in_len]; - auto* place = new paddle::platform::GPUPlace(); - - float* input_ptr = input_tmp.data(); - for (int i = 0; i < in_len; ++i) arr[i] = i; // rand() / double(RAND_MAX/2); - memcpy(input_ptr, arr, in_len * sizeof(float)); - input.CopyFrom(input_tmp, *place); - - input_ptr = input_tmp.data(); - for (int i = 0; i < in_len; ++i) arr[i] = 0; - memcpy(input_ptr, arr, in_len * sizeof(float)); - input_grad.CopyFrom(input_tmp, *place); - - // output - input_ptr = output_tmp.data(); - for (int i = 0; i < output_len; ++i) - arr[i] = 0; // rand() / double(RAND_MAX/2); - memcpy(input_ptr, arr, output_len * sizeof(float)); - output.CopyFrom(input_tmp, *place); - - // output - input_ptr = output_tmp.data(); - for (int i = 0; i < output_len; ++i) - arr[i] = 1; // rand() / double(RAND_MAX/2); - memcpy(input_ptr, arr, output_len * sizeof(float)); - output_grad.CopyFrom(input_tmp, *place); - - paddle::platform::DeviceContext* context = - new paddle::platform::CUDADeviceContext(paddle::platform::GPUPlace()); - paddle::operators::math::pool::maxPool pool_process; - paddle::operators::math::pool::maxPoolGrad poolGrad_process; - - testPool2d, - paddle::operators::math::pool::maxPoolGrad>( - *context, pool_process, poolGrad_process, input, input_grad, output, - output_grad, ksize, strides, paddings); -} - -int main() { - // testPool3d(); - test2dPool(); - // testPool3d(); -} -#endif \ No newline at end of file diff --git a/paddle/operators/math/pool_test_maxPool3d.cc b/paddle/operators/math/pool_test_maxPool3d.cc deleted file mode 100644 index ce572164c4..0000000000 --- a/paddle/operators/math/pool_test_maxPool3d.cc +++ /dev/null @@ -1,157 +0,0 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. */ - -#include -#include "paddle/operators/math/pooling.h" - -#include "paddle/memory/memcpy.h" -#include "paddle/platform/enforce.h" - -#include -#include - -#ifndef PADDLE_ONLY_CPU - -template -void testPool3d(paddle::platform::DeviceContext& context, PoolType pool_process, - PoolGradType poolGrad_process, paddle::framework::Tensor& input, - paddle::framework::Tensor& input_grad, - paddle::framework::Tensor& output, - paddle::framework::Tensor& output_grad, std::vector& ksize, - std::vector& strides, std::vector& paddings) { - paddle::operators::math::Pool3dForwardFunctor - pool3d_forward; - pool3d_forward(context, input, output, ksize, strides, paddings, - pool_process); - - int times = 50; - clock_t start, finish; - double totaltime; - - // Pool3dBackwardFunctor - start = clock(); - for (int i = 0; i < times; ++i) { - paddle::operators::math::Pool3dBackwardFunctor - pool3d_backward; - pool3d_backward(context, input, input_grad, output, output_grad, ksize, - strides, paddings, poolGrad_process); - PADDLE_ENFORCE(cudaStreamSynchronize(0), - "cudaStreamSynchronize failed in pool3d_backward CopyFrom"); - } - finish = clock(); - totaltime = (double)(finish - start) / CLOCKS_PER_SEC; - totaltime /= times; - std::cout << "\nPool3dBackwardFunctor: " << totaltime << "s" << std::endl; - - // MaxPool3dBackwardFunctor - start = clock(); - for (int j = 0; j < times; ++j) { - paddle::operators::math::MaxPool3dBackwardFunctor< - paddle::platform::GPUPlace, float> - maxpool3d_backward; - maxpool3d_backward(context, input, input_grad, output, output_grad, ksize, - strides, paddings); - PADDLE_ENFORCE( - cudaStreamSynchronize(0), - "cudaStreamSynchronize failed in maxpool3d_backward CopyFrom"); - } - finish = clock(); - totaltime = (double)(finish - start) / CLOCKS_PER_SEC; - totaltime /= times; - std::cout << "\nMaxPool3dBackwardFunctor: " << totaltime << "s" << std::endl; -} - -void test3dPool() { - using paddle::platform::DeviceContext; - using paddle::platform::CUDADeviceContext; - using paddle::platform::GPUPlace; - - paddle::framework::Tensor input_tmp; - paddle::framework::Tensor output_tmp; - paddle::framework::Tensor input; - paddle::framework::Tensor input_grad; - paddle::framework::Tensor output; - paddle::framework::Tensor output_grad; - - int batch = 32; - int channel = 4; - int input_depth = 4; - int input_height = 128; - int input_width = 128; - int in_len = batch * channel * input_depth * input_height * input_width; - std::vector ksize({3, 3, 3}); - std::vector strides({2, 2, 2}); - std::vector paddings({1, 1, 1}); - - int output_depth = - (input_depth - ksize[0] + 2 * paddings[0]) / strides[0] + 1; - int output_height = - (input_height - ksize[1] + 2 * paddings[1]) / strides[1] + 1; - int output_width = - (input_width - ksize[2] + 2 * paddings[2]) / strides[2] + 1; - - int output_len = output_depth * output_height * output_width; - - input_tmp.mutable_data( - {batch, channel, input_depth, input_height, input_width}, - paddle::platform::CPUPlace()); - output_tmp.mutable_data( - {batch, channel, output_depth, output_height, output_width}, - paddle::platform::CPUPlace()); - - float* arr = new float[in_len]; - auto* place = new paddle::platform::GPUPlace(); - - // input - float* input_ptr = input_tmp.data(); - for (int i = 0; i < in_len; ++i) arr[i] = i; // rand() / double(RAND_MAX/2); - memcpy(input_ptr, arr, in_len * sizeof(float)); - input.CopyFrom(input_tmp, *place); - - // input_grad - input_ptr = input_tmp.data(); - for (int i = 0; i < in_len; ++i) arr[i] = 0; - memcpy(input_ptr, arr, in_len * sizeof(float)); - input_grad.CopyFrom(input_tmp, *place); - - // output - input_ptr = output_tmp.data(); - for (int i = 0; i < output_len; ++i) - arr[i] = 0; // rand() / double(RAND_MAX/2); - memcpy(input_ptr, arr, output_len * sizeof(float)); - output.CopyFrom(input_tmp, *place); - - // output_grad - input_ptr = output_tmp.data(); - for (int i = 0; i < output_len; ++i) - arr[i] = 1; // rand() / double(RAND_MAX/2); - memcpy(input_ptr, arr, output_len * sizeof(float)); - output_grad.CopyFrom(input_tmp, *place); - - paddle::platform::DeviceContext* context = - new paddle::platform::CUDADeviceContext(paddle::platform::GPUPlace()); - paddle::operators::math::pool::maxPool pool_process; - - paddle::operators::math::pool::maxPoolGrad poolGrad_process; - - testPool3d, - paddle::operators::math::pool::maxPoolGrad>( - *context, pool_process, poolGrad_process, input, input_grad, output, - output_grad, ksize, strides, paddings); -} - -int main() { test3dPool(); } -#endif \ No newline at end of file diff --git a/paddle/operators/math/pooling.cc b/paddle/operators/math/pooling.cc index cb1f9d7285..1918c8b169 100644 --- a/paddle/operators/math/pooling.cc +++ b/paddle/operators/math/pooling.cc @@ -19,12 +19,12 @@ namespace operators { namespace math { template -class Pool2dForwardFunctor { +class Pool2dFunctor { public: void operator()(const platform::DeviceContext& context, const framework::Tensor& input, framework::Tensor& output, std::vector& ksize, std::vector& strides, - std::vector& paddings, PoolProcess pool_process) { + std::vector& paddings, PoolProcess pool_compute) { const int batch_size = input.dims()[0]; const int input_height = input.dims()[2]; const int input_width = input.dims()[3]; @@ -54,14 +54,14 @@ class Pool2dForwardFunctor { int wstart = pw * stride_width - padding_width; int wend = std::min(wstart + ksize_width, input_width); wstart = std::max(wstart, 0); - T ele = pool_process.initial(); + T ele = pool_compute.initial(); for (int h = hstart; h < hend; ++h) { for (int w = wstart; w < wend; ++w) { - pool_process.process(ele, input_data[h * input_width + w]); + pool_compute.compute(ele, input_data[h * input_width + w]); } } int pool_size = (hend - hstart) * (wend - wstart); - pool_process.finalize(ele, (static_cast(pool_size))); + pool_compute.finalize(ele, (static_cast(pool_size))); output_data[ph * output_width + pw] = ele; } } @@ -73,14 +73,14 @@ class Pool2dForwardFunctor { }; template -class Pool2dBackwardFunctor { +class Pool2dGradFunctor { public: void operator()(const platform::DeviceContext& context, const framework::Tensor& input, framework::Tensor& input_grad, const framework::Tensor& output, const framework::Tensor& output_grad, std::vector& ksize, std::vector& strides, std::vector& paddings, - PoolProcess pool_process) { + PoolProcess pool_compute) { const int batch_size = input.dims()[0]; const int input_height = input.dims()[2]; const int input_width = input.dims()[3]; @@ -115,12 +115,11 @@ class Pool2dBackwardFunctor { float scale = 1.0 / pool_size; for (int h = hstart; h < hend; ++h) { for (int w = wstart; w < wend; ++w) { - pool_process.gradProcess( - input_data[h * input_width + w], - output_data[ph * output_width + pw], - output_grad_data[ph * output_width + pw], - input_grad_data[h * input_width + w], - static_cast(scale)); + pool_compute.compute(input_data[h * input_width + w], + output_data[ph * output_width + pw], + output_grad_data[ph * output_width + pw], + input_grad_data[h * input_width + w], + static_cast(scale)); } } } @@ -135,7 +134,7 @@ class Pool2dBackwardFunctor { }; template -class MaxPool2dBackwardFunctor { +class MaxPool2dGradFunctor { public: void operator()(const platform::DeviceContext& context, const framework::Tensor& input, framework::Tensor& input_grad, @@ -195,37 +194,33 @@ class MaxPool2dBackwardFunctor { } }; -template class MaxPool2dBackwardFunctor; -// template class MaxPool2dBackwardFunctor; - -template class Pool2dForwardFunctor< - platform::CPUPlace, paddle::operators::math::pool::maxPool, float>; -template class Pool2dForwardFunctor< - platform::CPUPlace, paddle::operators::math::pool::avgPool, float>; -template class Pool2dBackwardFunctor< - platform::CPUPlace, paddle::operators::math::pool::maxPoolGrad, - float>; -template class Pool2dBackwardFunctor< - platform::CPUPlace, paddle::operators::math::pool::avgPoolGrad, - float>; -template class Pool2dForwardFunctor< - platform::CPUPlace, paddle::operators::math::pool::maxPool, double>; -template class Pool2dForwardFunctor< - platform::CPUPlace, paddle::operators::math::pool::avgPool, double>; -template class Pool2dBackwardFunctor< - platform::CPUPlace, paddle::operators::math::pool::maxPoolGrad, - double>; -template class Pool2dBackwardFunctor< - platform::CPUPlace, paddle::operators::math::pool::avgPoolGrad, - double>; +template class MaxPool2dGradFunctor; +// template class MaxPool2dGradFunctor; + +template class Pool2dFunctor, float>; +template class Pool2dFunctor, float>; +template class Pool2dGradFunctor< + platform::CPUPlace, paddle::operators::math::maxPoolGrad, float>; +template class Pool2dGradFunctor< + platform::CPUPlace, paddle::operators::math::avgPoolGrad, float>; +template class Pool2dFunctor, double>; +template class Pool2dFunctor, double>; +template class Pool2dGradFunctor< + platform::CPUPlace, paddle::operators::math::maxPoolGrad, double>; +template class Pool2dGradFunctor< + platform::CPUPlace, paddle::operators::math::avgPoolGrad, double>; template -class Pool3dForwardFunctor { +class Pool3dFunctor { public: void operator()(const platform::DeviceContext& context, const framework::Tensor& input, framework::Tensor& output, std::vector& ksize, std::vector& strides, - std::vector& paddings, PoolProcess pool_process) { + std::vector& paddings, PoolProcess pool_compute) { const int batch_size = input.dims()[0]; const int input_depth = input.dims()[2]; const int input_height = input.dims()[3]; @@ -265,11 +260,11 @@ class Pool3dForwardFunctor { int wend = std::min(wstart + ksize_width, input_width); wstart = std::max(wstart, 0); int output_idx = (pd * output_height + ph) * output_width + pw; - T ele = pool_process.initial(); + T ele = pool_compute.initial(); for (int d = dstart; d < dend; ++d) { for (int h = hstart; h < hend; ++h) { for (int w = wstart; w < wend; ++w) { - pool_process.process( + pool_compute.compute( ele, input_data[(d * input_height + h) * input_width + w]); } @@ -277,7 +272,7 @@ class Pool3dForwardFunctor { } int pool_size = (dend - dstart) * (hend - hstart) * (wend - wstart); - pool_process.finalize(ele, static_cast(pool_size)); + pool_compute.finalize(ele, static_cast(pool_size)); output_data[output_idx] = ele; } } @@ -290,14 +285,14 @@ class Pool3dForwardFunctor { }; template -class Pool3dBackwardFunctor { +class Pool3dGradFunctor { public: void operator()(const platform::DeviceContext& context, const framework::Tensor& input, framework::Tensor& input_grad, const framework::Tensor& output, const framework::Tensor& output_grad, std::vector& ksize, std::vector& strides, std::vector& paddings, - PoolProcess pool_process) { + PoolProcess pool_compute) { const int batch_size = input.dims()[0]; const int input_depth = input.dims()[2]; const int input_height = input.dims()[3]; @@ -348,7 +343,7 @@ class Pool3dBackwardFunctor { int input_idx = (d * input_height + h) * input_width + w; int output_idx = (pd * output_height + ph) * output_width + pw; - pool_process.gradProcess( + pool_compute.compute( input_data[input_idx], output_data[output_idx], output_grad_data[output_idx], input_grad_data[input_idx], static_cast(scale)); @@ -368,7 +363,7 @@ class Pool3dBackwardFunctor { }; template -class MaxPool3dBackwardFunctor { +class MaxPool3dGradFunctor { public: void operator()(const platform::DeviceContext& context, const framework::Tensor& input, framework::Tensor& input_grad, @@ -442,29 +437,25 @@ class MaxPool3dBackwardFunctor { } }; -template class MaxPool3dBackwardFunctor; -// template class MaxPool3dBackwardFunctor; - -template class Pool3dForwardFunctor< - platform::CPUPlace, paddle::operators::math::pool::maxPool, float>; -template class Pool3dForwardFunctor< - platform::CPUPlace, paddle::operators::math::pool::avgPool, float>; -template class Pool3dBackwardFunctor< - platform::CPUPlace, paddle::operators::math::pool::maxPoolGrad, - float>; -template class Pool3dBackwardFunctor< - platform::CPUPlace, paddle::operators::math::pool::avgPoolGrad, - float>; -template class Pool3dForwardFunctor< - platform::CPUPlace, paddle::operators::math::pool::maxPool, double>; -template class Pool3dForwardFunctor< - platform::CPUPlace, paddle::operators::math::pool::avgPool, double>; -template class Pool3dBackwardFunctor< - platform::CPUPlace, paddle::operators::math::pool::maxPoolGrad, - double>; -template class Pool3dBackwardFunctor< - platform::CPUPlace, paddle::operators::math::pool::avgPoolGrad, - double>; +template class MaxPool3dGradFunctor; +// template class MaxPool3dGradFunctor; + +template class Pool3dFunctor, float>; +template class Pool3dFunctor, float>; +template class Pool3dGradFunctor< + platform::CPUPlace, paddle::operators::math::maxPoolGrad, float>; +template class Pool3dGradFunctor< + platform::CPUPlace, paddle::operators::math::avgPoolGrad, float>; +template class Pool3dFunctor, double>; +template class Pool3dFunctor, double>; +template class Pool3dGradFunctor< + platform::CPUPlace, paddle::operators::math::maxPoolGrad, double>; +template class Pool3dGradFunctor< + platform::CPUPlace, paddle::operators::math::avgPoolGrad, double>; } // namespace math } // namespace operators } // namespace paddle diff --git a/paddle/operators/math/pooling.cu b/paddle/operators/math/pooling.cu index f41d80f77f..4164920035 100644 --- a/paddle/operators/math/pooling.cu +++ b/paddle/operators/math/pooling.cu @@ -25,7 +25,7 @@ __global__ void KernelPool2dForward( const int input_height, const int input_width, const int output_height, const int output_width, const int ksize_height, const int ksize_width, const int stride_height, const int stride_width, const int padding_height, - const int padding_width, PoolProcess pool_process) { + const int padding_width, PoolProcess pool_compute) { int index = blockIdx.x * blockDim.x + threadIdx.x; if (index < nthreads) { int pw = index % output_width; @@ -42,14 +42,14 @@ __global__ void KernelPool2dForward( wstart = max(wstart, 0); input_data += (batch_idx * channels + c) * input_height * input_width; - T ele = pool_process.initial(); + T ele = pool_compute.initial(); for (int h = hstart; h < hend; ++h) { for (int w = wstart; w < wend; ++w) { - pool_process.process(ele, input_data[h * input_width + w]); + pool_compute.compute(ele, input_data[h * input_width + w]); } } int pool_size = (hend - hstart) * (wend - wstart); - pool_process.finalize(ele, (static_cast(pool_size))); + pool_compute.finalize(ele, (static_cast(pool_size))); output_data[index] = ele; } } @@ -61,7 +61,7 @@ __global__ void KernelPool2dBackward( const int input_height, const int input_width, const int output_height, const int output_width, const int ksize_height, const int ksize_width, const int stride_height, const int stride_width, const int padding_height, - const int padding_width, PoolProcess pool_process) { + const int padding_width, PoolProcess pool_compute) { int index = blockIdx.x * blockDim.x + threadIdx.x; if (index < nthreads) { int offsetW = index % input_width + padding_width; @@ -93,9 +93,9 @@ __global__ void KernelPool2dBackward( wstart = max(wstart, 0); int pool_size = (hend - hstart) * (wend - wstart); int output_sub_idx = ph * output_width + pw; - pool_process.gradProcess(input, output_data[output_sub_idx], - output_grad[output_sub_idx], gradient, - static_cast(1.0 / pool_size)); + pool_compute.compute(input, output_data[output_sub_idx], + output_grad[output_sub_idx], gradient, + static_cast(1.0 / pool_size)); } } input_grad[index] = gradient; @@ -148,12 +148,12 @@ __global__ void KernelMaxPool2dBackward( } template -class Pool2dForwardFunctor { +class Pool2dFunctor { public: void operator()(const platform::DeviceContext& context, const framework::Tensor& input, framework::Tensor& output, std::vector& ksize, std::vector& strides, - std::vector& paddings, PoolProcess pool_process) { + std::vector& paddings, PoolProcess pool_compute) { const int batch_size = input.dims()[0]; const int input_channels = input.dims()[1]; const int input_height = input.dims()[2]; @@ -184,19 +184,19 @@ class Pool2dForwardFunctor { input_height, input_width, output_height, output_width, ksize_height, ksize_width, stride_height, stride_width, padding_height, - padding_width, pool_process); + padding_width, pool_compute); } }; template -class Pool2dBackwardFunctor { +class Pool2dGradFunctor { public: void operator()(const platform::DeviceContext& context, const framework::Tensor& input, framework::Tensor& input_grad, const framework::Tensor& output, const framework::Tensor& output_grad, std::vector& ksize, std::vector& strides, std::vector& paddings, - PoolProcess pool_process) { + PoolProcess pool_compute) { const int batch_size = input.dims()[0]; const int input_channels = input.dims()[1]; const int input_height = input.dims()[2]; @@ -228,12 +228,12 @@ class Pool2dBackwardFunctor { nthreads, input_data, output_data, output_grad_data, input_grad_data, input_channels, input_height, input_width, output_height, output_width, ksize_height, ksize_width, stride_height, stride_width, padding_height, - padding_width, pool_process); + padding_width, pool_compute); } }; template -class MaxPool2dBackwardFunctor { +class MaxPool2dGradFunctor { public: void operator()(const platform::DeviceContext& context, const framework::Tensor& input, framework::Tensor& input_grad, @@ -275,29 +275,25 @@ class MaxPool2dBackwardFunctor { } }; -template class MaxPool2dBackwardFunctor; -// template class MaxPool2dBackwardFunctor; - -template class Pool2dForwardFunctor< - platform::GPUPlace, paddle::operators::math::pool::maxPool, float>; -template class Pool2dForwardFunctor< - platform::GPUPlace, paddle::operators::math::pool::avgPool, float>; -template class Pool2dBackwardFunctor< - platform::GPUPlace, paddle::operators::math::pool::maxPoolGrad, - float>; -template class Pool2dBackwardFunctor< - platform::GPUPlace, paddle::operators::math::pool::avgPoolGrad, - float>; -template class Pool2dForwardFunctor< - platform::GPUPlace, paddle::operators::math::pool::maxPool, double>; -template class Pool2dForwardFunctor< - platform::GPUPlace, paddle::operators::math::pool::avgPool, double>; -template class Pool2dBackwardFunctor< - platform::GPUPlace, paddle::operators::math::pool::maxPoolGrad, - double>; -template class Pool2dBackwardFunctor< - platform::GPUPlace, paddle::operators::math::pool::avgPoolGrad, - double>; +template class MaxPool2dGradFunctor; +// template class MaxPool2dGradFunctor; + +template class Pool2dFunctor, float>; +template class Pool2dFunctor, float>; +template class Pool2dGradFunctor< + platform::GPUPlace, paddle::operators::math::maxPoolGrad, float>; +template class Pool2dGradFunctor< + platform::GPUPlace, paddle::operators::math::avgPoolGrad, float>; +template class Pool2dFunctor, double>; +template class Pool2dFunctor, double>; +template class Pool2dGradFunctor< + platform::GPUPlace, paddle::operators::math::maxPoolGrad, double>; +template class Pool2dGradFunctor< + platform::GPUPlace, paddle::operators::math::avgPoolGrad, double>; template __global__ void KernelPool3DForward( @@ -307,7 +303,7 @@ __global__ void KernelPool3DForward( const int ksize_depth, const int ksize_height, const int ksize_width, const int stride_depth, const int stride_height, const int stride_width, const int padding_depth, const int padding_height, const int padding_width, - PoolProcess pool_process) { + PoolProcess pool_compute) { for (int index = blockIdx.x * blockDim.x + threadIdx.x; index < (nthreads); index += blockDim.x * gridDim.x) { int pw = index % output_width; @@ -325,19 +321,19 @@ __global__ void KernelPool3DForward( dstart = max(dstart, 0); hstart = max(hstart, 0); wstart = max(wstart, 0); - T ele = pool_process.initial(); + T ele = pool_compute.initial(); input_data += (batch_idx * channels + c) * input_depth * input_height * input_width; for (int d = dstart; d < dend; ++d) { for (int h = hstart; h < hend; ++h) { for (int w = wstart; w < wend; ++w) { - pool_process.process( + pool_compute.compute( ele, input_data[(d * input_height + h) * input_width + w]); } } } int pool_size = (dend - dstart) * (hend - hstart) * (wend - wstart); - pool_process.finalize(ele, static_cast(pool_size)); + pool_compute.finalize(ele, static_cast(pool_size)); output_data[index] = ele; } } @@ -351,7 +347,7 @@ __global__ void KernelPool3DBackward( const int ksize_depth, const int ksize_height, const int ksize_width, const int stride_depth, const int stride_height, const int stride_width, const int padding_depth, const int padding_height, const int padding_width, - PoolProcess pool_process) { + PoolProcess pool_compute) { for (int index = blockIdx.x * blockDim.x + threadIdx.x; index < (nthreads); index += blockDim.x * gridDim.x) { int offsetW = index % input_width + padding_width; @@ -396,9 +392,9 @@ __global__ void KernelPool3DBackward( wstart = max(wstart, 0); int pool_size = (dend - dstart) * (hend - hstart) * (wend - wstart); int output_sub_idx = (pd * output_height + ph) * output_width + pw; - pool_process.gradProcess(input, output_data[output_sub_idx], - output_grad[output_sub_idx], gradient, - static_cast(1.0 / pool_size)); + pool_compute.compute(input, output_data[output_sub_idx], + output_grad[output_sub_idx], gradient, + static_cast(1.0 / pool_size)); } } } @@ -459,12 +455,12 @@ __global__ void KernelMaxPool3DBackward( } template -class Pool3dForwardFunctor { +class Pool3dFunctor { public: void operator()(const platform::DeviceContext& context, const framework::Tensor& input, framework::Tensor& output, std::vector& ksize, std::vector& strides, - std::vector& paddings, PoolProcess pool_process) { + std::vector& paddings, PoolProcess pool_compute) { const int batch_size = input.dims()[0]; const int input_channels = input.dims()[1]; const int input_depth = input.dims()[2]; @@ -502,19 +498,19 @@ class Pool3dForwardFunctor { input_height, input_width, output_depth, output_height, output_width, ksize_depth, ksize_height, ksize_width, stride_depth, stride_height, stride_width, padding_depth, padding_height, padding_width, - pool_process); + pool_compute); } }; template -class Pool3dBackwardFunctor { +class Pool3dGradFunctor { public: void operator()(const platform::DeviceContext& context, const framework::Tensor& input, framework::Tensor& input_grad, const framework::Tensor& output, const framework::Tensor& output_grad, std::vector& ksize, std::vector& strides, std::vector& paddings, - PoolProcess pool_process) { + PoolProcess pool_compute) { const int batch_size = input.dims()[0]; const int input_channels = input.dims()[1]; const int input_depth = input.dims()[2]; @@ -554,12 +550,12 @@ class Pool3dBackwardFunctor { input_channels, input_depth, input_height, input_width, output_depth, output_height, output_width, ksize_depth, ksize_height, ksize_width, stride_depth, stride_height, stride_width, padding_depth, - padding_height, padding_width, pool_process); + padding_height, padding_width, pool_compute); } }; template -class MaxPool3dBackwardFunctor { +class MaxPool3dGradFunctor { public: void operator()(const platform::DeviceContext& context, const framework::Tensor& input, framework::Tensor& input_grad, @@ -608,29 +604,25 @@ class MaxPool3dBackwardFunctor { } }; -template class MaxPool3dBackwardFunctor; -// template class MaxPool3dBackwardFunctor; - -template class Pool3dForwardFunctor< - platform::GPUPlace, paddle::operators::math::pool::maxPool, float>; -template class Pool3dForwardFunctor< - platform::GPUPlace, paddle::operators::math::pool::avgPool, float>; -template class Pool3dBackwardFunctor< - platform::GPUPlace, paddle::operators::math::pool::maxPoolGrad, - float>; -template class Pool3dBackwardFunctor< - platform::GPUPlace, paddle::operators::math::pool::avgPoolGrad, - float>; -template class Pool3dForwardFunctor< - platform::GPUPlace, paddle::operators::math::pool::maxPool, double>; -template class Pool3dForwardFunctor< - platform::GPUPlace, paddle::operators::math::pool::avgPool, double>; -template class Pool3dBackwardFunctor< - platform::GPUPlace, paddle::operators::math::pool::maxPoolGrad, - double>; -template class Pool3dBackwardFunctor< - platform::GPUPlace, paddle::operators::math::pool::avgPoolGrad, - double>; +template class MaxPool3dGradFunctor; +// template class MaxPool3dGradFunctor; + +template class Pool3dFunctor, float>; +template class Pool3dFunctor, float>; +template class Pool3dGradFunctor< + platform::GPUPlace, paddle::operators::math::maxPoolGrad, float>; +template class Pool3dGradFunctor< + platform::GPUPlace, paddle::operators::math::avgPoolGrad, float>; +template class Pool3dFunctor, double>; +template class Pool3dFunctor, double>; +template class Pool3dGradFunctor< + platform::GPUPlace, paddle::operators::math::maxPoolGrad, double>; +template class Pool3dGradFunctor< + platform::GPUPlace, paddle::operators::math::avgPoolGrad, double>; } // namespace math } // namespace operators diff --git a/paddle/operators/math/pooling.h b/paddle/operators/math/pooling.h index 2e05a5968a..cf0ab7ecae 100644 --- a/paddle/operators/math/pooling.h +++ b/paddle/operators/math/pooling.h @@ -21,17 +21,15 @@ limitations under the License. */ namespace paddle { namespace operators { namespace math { - ////////////////////// #define FLT_MAX __FLT_MAX__ ///////////////////// -namespace pool { template class maxPool { public: DEVICE inline T initial() { return static_cast(-FLT_MAX); } - DEVICE inline void process(T& y, const T& x) { y = y > x ? y : x; } + DEVICE inline void compute(T& y, const T& x) { y = y > x ? y : x; } DEVICE inline void finalize(T& y, const T& poo_size) {} }; @@ -39,14 +37,14 @@ template class avgPool { public: DEVICE inline T initial() { return static_cast(0); } - DEVICE inline void process(T& y, const T& x) { y += x; } + DEVICE inline void compute(T& y, const T& x) { y += x; } DEVICE inline void finalize(T& y, const T& poo_size) { y /= poo_size; } }; template class maxPoolGrad { public: - DEVICE inline void gradProcess(const T& x, const T& y, const T& dy, T& dx, - T scale) { + DEVICE inline void compute(const T& x, const T& y, const T& dy, T& dx, + T scale) { dx += dy * (x == y); } }; @@ -54,35 +52,34 @@ class maxPoolGrad { template class avgPoolGrad { public: - DEVICE inline void gradProcess(const T& x, const T& y, const T& dy, T& dx, - T scale) { + DEVICE inline void compute(const T& x, const T& y, const T& dy, T& dx, + T scale) { dx += (scale * dy); } }; -} // namespace pool template -class Pool2dForwardFunctor { +class Pool2dFunctor { public: void operator()(const platform::DeviceContext& context, const framework::Tensor& input, framework::Tensor& output, std::vector& ksize, std::vector& strides, - std::vector& paddings, PoolProcess pool_process); + std::vector& paddings, PoolProcess pool_compute); }; template -class Pool2dBackwardFunctor { +class Pool2dGradFunctor { public: void operator()(const platform::DeviceContext& context, const framework::Tensor& input, framework::Tensor& input_grad, const framework::Tensor& output, const framework::Tensor& output_grad, std::vector& ksize, std::vector& strides, std::vector& paddings, - PoolProcess pool_process); + PoolProcess pool_compute); }; template -class MaxPool2dBackwardFunctor { +class MaxPool2dGradFunctor { public: void operator()(const platform::DeviceContext& context, const framework::Tensor& input, framework::Tensor& input_grad, @@ -92,27 +89,27 @@ class MaxPool2dBackwardFunctor { }; template -class Pool3dForwardFunctor { +class Pool3dFunctor { public: void operator()(const platform::DeviceContext& context, const framework::Tensor& input, framework::Tensor& output, std::vector& ksize, std::vector& strides, - std::vector& paddings, PoolProcess pool_process); + std::vector& paddings, PoolProcess pool_compute); }; template -class Pool3dBackwardFunctor { +class Pool3dGradFunctor { public: void operator()(const platform::DeviceContext& context, const framework::Tensor& input, framework::Tensor& input_grad, const framework::Tensor& output, const framework::Tensor& output_grad, std::vector& ksize, std::vector& strides, std::vector& paddings, - PoolProcess pool_process); + PoolProcess pool_compute); }; template -class MaxPool3dBackwardFunctor { +class MaxPool3dGradFunctor { public: void operator()(const platform::DeviceContext& context, const framework::Tensor& input, framework::Tensor& input_grad, diff --git a/paddle/operators/pool_op.cc b/paddle/operators/pool_op.cc index 9c1b7ea41d..a5e731cc66 100644 --- a/paddle/operators/pool_op.cc +++ b/paddle/operators/pool_op.cc @@ -17,7 +17,7 @@ limitations under the License. */ namespace paddle { namespace operators { -int outputSize_pool(int input_size, int filter_size, int padding, int stride) { +int OutputSizePool(int input_size, int filter_size, int padding, int stride) { int output_size = (input_size - filter_size + 2 * padding) / stride + 1; return output_size; } @@ -33,7 +33,7 @@ class PoolOp : public framework::OperatorWithKernel { PADDLE_ENFORCE_NOT_NULL(ctx.OutputVar("Out"), "Out(Output) of Pooling should not be null."); - auto in_X = ctx.Input("X"); + auto in_x = ctx.Input("X"); auto out = ctx.Output("Out"); int global_pooling = Attr("globalPooling"); std::string pooling_type = Attr("poolingType"); @@ -43,30 +43,25 @@ class PoolOp : public framework::OperatorWithKernel { PADDLE_ENFORCE(pooling_type == "max" || pooling_type == "avg", "pooling_type should be 'max' or 'avg'"); - PADDLE_ENFORCE(in_X->dims().size() == 4 || in_X->dims().size() == 5, + PADDLE_ENFORCE(in_x->dims().size() == 4 || in_x->dims().size() == 5, "Pooling intput should be 4-D or 5-D"); + PADDLE_ENFORCE(ksize.size() == 2 || ksize.size() == 3, + "Pooling size should be 2 elements. or 3 elements."); + PADDLE_ENFORCE_EQ(ksize.size(), strides.size(), + "strides size and pooling size should be the same."); + PADDLE_ENFORCE_EQ(ksize.size(), paddings.size(), + "paddings size and pooling size should be the same."); if (global_pooling == 1) { - ksize.resize(static_cast(in_X->dims().size()) - 2); + ksize.resize(static_cast(in_x->dims().size()) - 2); for (size_t i = 0; i < ksize.size(); ++i) - ksize[i] = static_cast(in_X->dims()[i + 2]); + ksize[i] = static_cast(in_x->dims()[i + 2]); } - if (ksize.size() == 2) { - PADDLE_ENFORCE_EQ(strides.size(), 2, - "Pool2DOp strides size should be 2 elements."); - PADDLE_ENFORCE_EQ(paddings.size(), 2, - "Pool2DOp paddings size should be 2 elements"); - } else { - PADDLE_ENFORCE_EQ(strides.size(), 3, - "Pool3DOp strides should be 3 elements."); - PADDLE_ENFORCE_EQ(paddings.size(), 3, - "Pool3DOp paddings should be 3 elements."); - } - std::vector output_shape({in_X->dims()[0], in_X->dims()[1]}); + std::vector output_shape({in_x->dims()[0], in_x->dims()[1]}); for (size_t i = 0; i < ksize.size(); ++i) { - output_shape.push_back(outputSize_pool(in_X->dims()[i + 2], ksize[i], - paddings[i], strides[i])); + output_shape.push_back(OutputSizePool(in_x->dims()[i + 2], ksize[i], + paddings[i], strides[i])); } out->Resize(framework::make_ddim(output_shape)); } @@ -78,9 +73,16 @@ class PoolOpGrad : public framework::OperatorWithKernel { protected: void InferShape(const framework::InferShapeContext &ctx) const override { + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("X"), + "X(Input) of Pooling should not be null."); + PADDLE_ENFORCE_NOT_NULL(ctx.InputVar("Out"), + "Out(Output) of Pooling should not be null."); + PADDLE_ENFORCE_NOT_NULL(ctx.Output(framework::GradVarName("X")), + "Input@Grad of Pooling should not be null."); + auto in = ctx.Input("X"); auto d_in = ctx.Output(framework::GradVarName("X")); - if (d_in) d_in->Resize(in->dims()); + d_in->Resize(in->dims()); } }; @@ -92,7 +94,7 @@ class Pool2dOpMaker : public framework::OpProtoAndCheckerMaker { "X", "The input tensor of pooling operator. " "The format of input tensor is NCHW. Where N is batch size, C is the " - "number of channels, H and W is the height and width of image."); + "number of channels, H and W is the height and width of feature."); AddOutput("Out", "The output tensor of pooling operator." "The format of output tensor is also NCHW."); @@ -166,7 +168,7 @@ class Pool3dOpMaker : public framework::OpProtoAndCheckerMaker { "The format of input tensor is NCDHW. Where N is batch size, C is " "the " "number of channels, D, H and W is the depth, height and width of " - "image."); + "feature."); AddOutput("Out", "The output tensor of pooling operator." "The format of output tensor is also NCDHW."); diff --git a/paddle/operators/pool_op.cu b/paddle/operators/pool_op.cu index b011d20da5..0e3b80868f 100644 --- a/paddle/operators/pool_op.cu +++ b/paddle/operators/pool_op.cu @@ -1,15 +1,16 @@ -/* Copyright (c) 2016 PaddlePaddle Authors All Rights Reserve. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - http://www.apache.org/licenses/LICENSE-2.0 +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ #include "paddle/operators/pool_op.h" diff --git a/paddle/operators/pool_op.h b/paddle/operators/pool_op.h index 763103d884..9471282205 100644 --- a/paddle/operators/pool_op.h +++ b/paddle/operators/pool_op.h @@ -28,7 +28,7 @@ template class PoolKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { - const Tensor* in_X = context.Input("X"); + const Tensor* in_x = context.Input("X"); Tensor* out = context.Output("Out"); int global_pooling = context.Attr("globalPooling"); @@ -38,43 +38,43 @@ class PoolKernel : public framework::OpKernel { std::vector paddings = context.Attr>("paddings"); if (global_pooling == 1) { for (size_t i = 0; i < ksize.size(); ++i) { - ksize[i] = in_X->dims()[i + 2]; + ksize[i] = static_cast(in_x->dims()[i + 2]); } } switch (ksize.size()) { case 2: { if (pooling_type == "max") { - paddle::operators::math::Pool2dForwardFunctor< - Place, paddle::operators::math::pool::maxPool, T> + paddle::operators::math::Pool2dFunctor< + Place, paddle::operators::math::maxPool, T> pool2d_forward; - paddle::operators::math::pool::maxPool pool_process; - pool2d_forward(context.device_context(), *in_X, *out, ksize, strides, + paddle::operators::math::maxPool pool_process; + pool2d_forward(context.device_context(), *in_x, *out, ksize, strides, paddings, pool_process); } else if (pooling_type == "avg") { - paddle::operators::math::Pool2dForwardFunctor< - Place, paddle::operators::math::pool::avgPool, T> + paddle::operators::math::Pool2dFunctor< + Place, paddle::operators::math::avgPool, T> pool2d_forward; - paddle::operators::math::pool::avgPool pool_process; - pool2d_forward(context.device_context(), *in_X, *out, ksize, strides, + paddle::operators::math::avgPool pool_process; + pool2d_forward(context.device_context(), *in_x, *out, ksize, strides, paddings, pool_process); } } break; case 3: { if (pooling_type == "max") { - paddle::operators::math::Pool3dForwardFunctor< - Place, paddle::operators::math::pool::maxPool, T> + paddle::operators::math::Pool3dFunctor< + Place, paddle::operators::math::maxPool, T> pool3d_forward; - paddle::operators::math::pool::maxPool pool_process; - pool3d_forward(context.device_context(), *in_X, *out, ksize, strides, + paddle::operators::math::maxPool pool_process; + pool3d_forward(context.device_context(), *in_x, *out, ksize, strides, paddings, pool_process); } else if (pooling_type == "avg") { - paddle::operators::math::Pool3dForwardFunctor< - Place, paddle::operators::math::pool::avgPool, T> + paddle::operators::math::Pool3dFunctor< + Place, paddle::operators::math::avgPool, T> pool3d_forward; - paddle::operators::math::pool::avgPool pool_process; - pool3d_forward(context.device_context(), *in_X, *out, ksize, strides, + paddle::operators::math::avgPool pool_process; + pool3d_forward(context.device_context(), *in_x, *out, ksize, strides, paddings, pool_process); } } break; @@ -86,11 +86,11 @@ template class PoolGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& context) const override { - const Tensor* in_X = context.Input("X"); + const Tensor* in_x = context.Input("X"); const Tensor* out = context.Input("Out"); const Tensor* out_grad = context.Input(framework::GradVarName("Out")); - Tensor* in_X_grad = context.Output(framework::GradVarName("X")); + Tensor* in_x_grad = context.Output(framework::GradVarName("X")); int global_pooling = context.Attr("globalPooling"); std::string pooling_type = context.Attr("poolingType"); @@ -99,43 +99,44 @@ class PoolGradKernel : public framework::OpKernel { std::vector paddings = context.Attr>("paddings"); if (global_pooling == 1) { - for (size_t i = 0; i < ksize.size(); ++i) ksize[i] = in_X->dims()[i + 2]; + for (size_t i = 0; i < ksize.size(); ++i) + ksize[i] = static_cast(in_x->dims()[i + 2]); } - if (in_X_grad) { - in_X_grad->mutable_data(context.GetPlace()); - auto temp = framework::EigenVector::Flatten(*in_X_grad); + if (in_x_grad) { + in_x_grad->mutable_data(context.GetPlace()); + auto temp = framework::EigenVector::Flatten(*in_x_grad); temp.device(context.GetEigenDevice()) = temp.constant(static_cast(0)); switch (ksize.size()) { case 2: { if (pooling_type == "max") { - paddle::operators::math::MaxPool2dBackwardFunctor + paddle::operators::math::MaxPool2dGradFunctor pool2d_backward; - pool2d_backward(context.device_context(), *in_X, *in_X_grad, *out, + pool2d_backward(context.device_context(), *in_x, *in_x_grad, *out, *out_grad, ksize, strides, paddings); } else if (pooling_type == "avg") { - paddle::operators::math::Pool2dBackwardFunctor< - Place, paddle::operators::math::pool::avgPoolGrad, T> + paddle::operators::math::Pool2dGradFunctor< + Place, paddle::operators::math::avgPoolGrad, T> pool2d_backward; - paddle::operators::math::pool::avgPoolGrad pool_process; - pool2d_backward(context.device_context(), *in_X, *in_X_grad, *out, + paddle::operators::math::avgPoolGrad pool_process; + pool2d_backward(context.device_context(), *in_x, *in_x_grad, *out, *out_grad, ksize, strides, paddings, pool_process); } } break; case 3: { if (pooling_type == "max") { - paddle::operators::math::MaxPool3dBackwardFunctor + paddle::operators::math::MaxPool3dGradFunctor pool3d_backward; - pool3d_backward(context.device_context(), *in_X, *in_X_grad, *out, + pool3d_backward(context.device_context(), *in_x, *in_x_grad, *out, *out_grad, ksize, strides, paddings); } else if (pooling_type == "avg") { - paddle::operators::math::Pool3dBackwardFunctor< - Place, paddle::operators::math::pool::avgPoolGrad, T> + paddle::operators::math::Pool3dGradFunctor< + Place, paddle::operators::math::avgPoolGrad, T> pool3d_backward; - paddle::operators::math::pool::avgPoolGrad pool_process; - pool3d_backward(context.device_context(), *in_X, *in_X_grad, *out, + paddle::operators::math::avgPoolGrad pool_process; + pool3d_backward(context.device_context(), *in_x, *in_x_grad, *out, *out_grad, ksize, strides, paddings, pool_process); } } break; From 4b06d8db9179c74a35582c85f782e8c268d361a6 Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Tue, 26 Sep 2017 20:06:12 +0800 Subject: [PATCH 029/170] fix globalPooling type (int => bool) --- paddle/operators/pool_op.cc | 47 ++++++++++++++++++++----------------- paddle/operators/pool_op.h | 8 +++---- 2 files changed, 30 insertions(+), 25 deletions(-) diff --git a/paddle/operators/pool_op.cc b/paddle/operators/pool_op.cc index a5e731cc66..9959b3ec07 100644 --- a/paddle/operators/pool_op.cc +++ b/paddle/operators/pool_op.cc @@ -35,7 +35,7 @@ class PoolOp : public framework::OperatorWithKernel { auto in_x = ctx.Input("X"); auto out = ctx.Output("Out"); - int global_pooling = Attr("globalPooling"); + bool global_pooling = Attr("globalPooling"); std::string pooling_type = Attr("poolingType"); std::vector ksize = Attr>("ksize"); std::vector strides = Attr>("strides"); @@ -45,6 +45,15 @@ class PoolOp : public framework::OperatorWithKernel { "pooling_type should be 'max' or 'avg'"); PADDLE_ENFORCE(in_x->dims().size() == 4 || in_x->dims().size() == 5, "Pooling intput should be 4-D or 5-D"); + + if (global_pooling) { + ksize.resize(static_cast(in_x->dims().size()) - 2); + for (size_t i = 0; i < ksize.size(); ++i) + ksize[i] = static_cast(in_x->dims()[i + 2]); + } + + PADDLE_ENFORCE(in_x->dims().size() == static_cast(ksize.size() + 2), + "Input size and Pooling size should be consistent."); PADDLE_ENFORCE(ksize.size() == 2 || ksize.size() == 3, "Pooling size should be 2 elements. or 3 elements."); PADDLE_ENFORCE_EQ(ksize.size(), strides.size(), @@ -52,12 +61,6 @@ class PoolOp : public framework::OperatorWithKernel { PADDLE_ENFORCE_EQ(ksize.size(), paddings.size(), "paddings size and pooling size should be the same."); - if (global_pooling == 1) { - ksize.resize(static_cast(in_x->dims().size()) - 2); - for (size_t i = 0; i < ksize.size(); ++i) - ksize[i] = static_cast(in_x->dims()[i + 2]); - } - std::vector output_shape({in_x->dims()[0], in_x->dims()[1]}); for (size_t i = 0; i < ksize.size(); ++i) { output_shape.push_back(OutputSizePool(in_x->dims()[i + 2], ksize[i], @@ -103,15 +106,16 @@ class Pool2dOpMaker : public framework::OpProtoAndCheckerMaker { "poolingType of pooling operator." "str constant equal to 'max' or 'avg'"); AddAttr>( - "ksize", "pooling size(height, width) of pooling operator.") - .AddCustomChecker(GreaterThanChecker_pool({0, 0})); - AddAttr( + "ksize", + "Pooling size(depth, height, width) of pooling operator." + "If globalPooling = true, ksize is ignored and need not be specified."); + AddAttr( "globalPooling", "whether to use the globalPooling." - "int constant equal to 0 or 1" - "default 0" - "If globalPooling = 1, ksize is ignored and need not be specified.") - .SetDefault(0); + "int constant equal to false or true" + "default false" + "If globalPooling = true, ksize is ignored and need not be specified.") + .SetDefault(false); AddAttr>("strides", "strides(height, width) of pooling operator." "default {1,1}") @@ -177,15 +181,16 @@ class Pool3dOpMaker : public framework::OpProtoAndCheckerMaker { "poolingType of pooling operator." "str constant equal to 'max' or 'avg'"); AddAttr>( - "ksize", "pooling size(depth, height, width) of pooling operator.") - .AddCustomChecker(GreaterThanChecker_pool({0, 0, 0})); - AddAttr( + "ksize", + "pooling size(depth, height, width) of pooling operator." + "If globalPooling = true, ksize is ignored and need not be specified."); + AddAttr( "globalPooling", "whether to use the globalPooling." - "int constant equal to 0 or 1" - "default 0" - "If globalPooling = 1, ksize is ignored and need not be specified.") - .SetDefault(0); + "int constant equal to false or true" + "default false" + "If globalPooling = true, ksize is ignored and need not be specified.") + .SetDefault(false); AddAttr>( "strides", "strides(depth, height, width) of pooling operator." diff --git a/paddle/operators/pool_op.h b/paddle/operators/pool_op.h index 9471282205..73c9721624 100644 --- a/paddle/operators/pool_op.h +++ b/paddle/operators/pool_op.h @@ -31,12 +31,12 @@ class PoolKernel : public framework::OpKernel { const Tensor* in_x = context.Input("X"); Tensor* out = context.Output("Out"); - int global_pooling = context.Attr("globalPooling"); + bool global_pooling = context.Attr("globalPooling"); std::string pooling_type = context.Attr("poolingType"); std::vector ksize = context.Attr>("ksize"); std::vector strides = context.Attr>("strides"); std::vector paddings = context.Attr>("paddings"); - if (global_pooling == 1) { + if (global_pooling) { for (size_t i = 0; i < ksize.size(); ++i) { ksize[i] = static_cast(in_x->dims()[i + 2]); } @@ -92,13 +92,13 @@ class PoolGradKernel : public framework::OpKernel { context.Input(framework::GradVarName("Out")); Tensor* in_x_grad = context.Output(framework::GradVarName("X")); - int global_pooling = context.Attr("globalPooling"); + bool global_pooling = context.Attr("globalPooling"); std::string pooling_type = context.Attr("poolingType"); std::vector ksize = context.Attr>("ksize"); std::vector strides = context.Attr>("strides"); std::vector paddings = context.Attr>("paddings"); - if (global_pooling == 1) { + if (global_pooling) { for (size_t i = 0; i < ksize.size(); ++i) ksize[i] = static_cast(in_x->dims()[i + 2]); } From 1c710053292394154d41db6c44ea808a8feaf65c Mon Sep 17 00:00:00 2001 From: Helin Wang Date: Sun, 10 Sep 2017 15:03:58 -0700 Subject: [PATCH 030/170] Design Doc: Session --- doc/design/session.md | 62 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 doc/design/session.md diff --git a/doc/design/session.md b/doc/design/session.md new file mode 100644 index 0000000000..2e8c0ece7a --- /dev/null +++ b/doc/design/session.md @@ -0,0 +1,62 @@ +# Design Doc: Session + +## Abstract + +This design doc proposes to have an object called *Session* which +encapsulates the environment in which the computation graph is +executed. + +## Background + +A computation graph is executed in an environment which contains the +[scope](./scope.md) and other states. PaddlePaddle used to only have +an implicit global session on which `paddle.eval()` is executed. + +This has the limitation that the user can not create two independent +environments. For example, in reinforcement learning, the user may +want to have a stale model for inference and a fresh model for +training, and only replace the stale model with the fresh model +periodically. Also, we have no concept that can encapsulate a remote +environment that could execute a computation graph. + +## Session + +Session is an object that owns all runtime states such as scope, +reader OP's file handles, connection to a remote PaddlePaddle cluster, +etc. + +Session has two methods: `eval` and `close`. `eval` executes the +target OP in a given graph, and `close` closes the session and +releases all related resources: + +```Python +a = paddle.constant(1.0) +b = paddle.constant(2.0) +c = a + b +sess = paddle.session() +sess.eval(c) +sess.close() +``` + +### Remote Session + +Paddle Cloud will support user creating a remote session pointing to +the Paddle Cloud cluster. The user can send the computation graph to +be executed on the Paddle Cloud. In this way, the user can control a +cluster from her local computer: + +```Python +reader = paddle.reader.recordio("/pfs/home/peter/mnist-train-*") # data stored on Paddle Cloud +image = reader.column(0) +label = reader.column(1) +fc1 = paddle.op.fc(image, size=256, act="sigmoid") +fc2 = paddle.op.fc(fc1, size=10, act="softmax") +cost = paddle.op.cross_entropy(fc2) +opt = paddle.optimizer.sgd(cost) + +remote_config = ... # remote configuration such as endpoint, number of nodes and authentication. +sess = paddle.remoteSession(remote_config) +for i in range(1000): + sess.eval(opt) +sess.close() +``` From 94dfd8649e06108bc0c03e6f53eb43ab13f30332 Mon Sep 17 00:00:00 2001 From: Helin Wang Date: Mon, 25 Sep 2017 16:02:58 -0700 Subject: [PATCH 031/170] fix according to comments --- doc/design/session.md | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/doc/design/session.md b/doc/design/session.md index 2e8c0ece7a..dc034c3906 100644 --- a/doc/design/session.md +++ b/doc/design/session.md @@ -6,26 +6,36 @@ This design doc proposes to have an object called *Session* which encapsulates the environment in which the computation graph is executed. +The session is able to distinguish running a graph locally or +remotely, using CPU only or using one or more GPUs. Different sessions +have different runtime environments such as [scopes](./scope.md) and +device contexts. + + ## Background -A computation graph is executed in an environment which contains the -[scope](./scope.md) and other states. PaddlePaddle used to only have -an implicit global session on which `paddle.eval()` is executed. +A computation graph runs in an environment which contains states such +as the scope and device contexts. The current design has an implicit +global session on which `paddle.eval()` is executed. + +Since the user is not able to explicitly switch between runtime +environments such as the scope and the device contexts, the user +cannot run a topology in two independent environments. For example, in +reinforcement learning, the user may want to have a stale model for +inference and a fresh model for training, and only replace the stale +model with the fresh model periodically. Also, we have no concept that +can encapsulate a remote environment that could execute a computation +graph. -This has the limitation that the user can not create two independent -environments. For example, in reinforcement learning, the user may -want to have a stale model for inference and a fresh model for -training, and only replace the stale model with the fresh model -periodically. Also, we have no concept that can encapsulate a remote -environment that could execute a computation graph. +We need a session concept to address above issues. ## Session -Session is an object that owns all runtime states such as scope, +A session is an object that owns all runtime states such as scope, reader OP's file handles, connection to a remote PaddlePaddle cluster, etc. -Session has two methods: `eval` and `close`. `eval` executes the +The session has two methods: `eval` and `close`. `eval` executes the target OP in a given graph, and `close` closes the session and releases all related resources: @@ -51,7 +61,7 @@ image = reader.column(0) label = reader.column(1) fc1 = paddle.op.fc(image, size=256, act="sigmoid") fc2 = paddle.op.fc(fc1, size=10, act="softmax") -cost = paddle.op.cross_entropy(fc2) +cost = paddle.op.cross_entropy(fc2, label) opt = paddle.optimizer.sgd(cost) remote_config = ... # remote configuration such as endpoint, number of nodes and authentication. From f24b5dffc42063ddfe28229fdb242c3df4ec1aa7 Mon Sep 17 00:00:00 2001 From: Helin Wang Date: Tue, 26 Sep 2017 18:03:37 -0700 Subject: [PATCH 032/170] Update Session design doc --- doc/design/refactor/session.md | 160 +++++++++++++++++++++++++++++++++ doc/design/session.md | 72 --------------- 2 files changed, 160 insertions(+), 72 deletions(-) create mode 100644 doc/design/refactor/session.md delete mode 100644 doc/design/session.md diff --git a/doc/design/refactor/session.md b/doc/design/refactor/session.md new file mode 100644 index 0000000000..5f58148f01 --- /dev/null +++ b/doc/design/refactor/session.md @@ -0,0 +1,160 @@ +# Design Doc: Session + +## Abstract + +The *session* object encapsulates the environment in which the +computation graph is executed. + +We will have *local* session and *remote* session, they offer the +same [interface](#interface). The local session encapsulates the local +runtime environment and the remote session encapsulates the cluster +runtime envrionment. + +The local runtime envrionment contains: + +1. computation devices (i.e., CPU, GPU) handles, and +1. the [scope](../scope.md) which holds all variables. + +The remote runtime envrionment contains: + +1. computation devices (i.e., CPU and GPU on node 0, 1) in a cluster, + and +1. the distributed [scope](../scope.md) in a cluster which holds all + variables. + +The user can create a remote session on Paddle Cloud and evaluate the +computation graph with it. In this way, the user can control the +remote computation resource in a cluster from his local computer. + + +## Background + +The current design has an implicit global session on which +`paddle.eval()` is executed. The pain point is: + +Since the user is not able to explicitly switch between runtime +environments such as the scope and the device contexts, the user +cannot run a topology in two independent environments. + +For example, in reinforcement learning, the user may want to have a +stale model for inference and a fresh model for training, and only +replace the stale model with the fresh model periodically. + +Furthermore, we have no concept that encapsulates a remote environment +that executes a computation graph. + +We need the session object to address above issues. + + +## Session + +A session is an object that owns the runtime environment. All +computations are executed through `session.eval`. + + +### Interface + +``` +eval( + targets, + feed_dict=None, +) +``` + +Evaluates the target Operations or Variables in `targets`. + +- *targets*: the evaluation targets. Can be a single Operation or + Variable, or a list with the Operations or Variables as elements. + + The value returned by `eval()` has the same shape as the `target` + argument. + + The computation graph is implicitly inferred from the targets. + +- *feed_dict*: a dictionary that contains the tensors which overrides + the edges of the computation graph. + +``` +close() +``` + +Closes the session. Calling this method releases the scope. + + +### Create a Local Session + +``` +session( + gpu_ids=None +) +``` + +Creates a new session. One session owns one scope, so creating +multiple sessions will create different scopes. + +- *gpu_ids*: a single `int` or a list of `int` of the GPU IDs to be + used as the computation devices. If not specified, all avaiable GPUs + will be used. + + +#### Example + +```Python +a = paddle.constant(1.0) +b = paddle.constant(2.0) +c = a + b +sess = paddle.session(gpu_ids=[0,1]) +sess.eval(c) +sess.close() +``` + +### Create a Remote Session + +``` +create_cloud_job( + name, + num_trainer, + mem_per_trainer, + gpu_per_trainer, + cpu_per_trainer, + num_ps, + mem_per_ps, + cpu_per_ps, +) +``` + +Creates a Paddle Cloud job. Fails if the job name exists. + +``` +get_cloud_job( + name +) +``` + +Gets a Paddle Cloud job. + +``` +remote_session( + job +) +``` + +- *job*: the Paddle Cloud job. + +#### Example + +```Python +reader = paddle.reader.recordio("/pfs/home/peter/mnist-train-*") # data stored on Paddle Cloud +image = reader.column(0) +label = reader.column(1) +fc1 = paddle.op.fc(image, size=256, act="sigmoid") +fc2 = paddle.op.fc(fc1, size=10, act="softmax") +cost = paddle.op.cross_entropy(fc2, label) +opt = paddle.optimizer.sgd(cost) + +job = paddle.create_cloud_job("test", 3, "1G", 1, 1, 2, "1G", 1) +sess = paddle.remote_ession(job) +for i in range(1000): + sess.eval(opt) +sess.close() +``` diff --git a/doc/design/session.md b/doc/design/session.md deleted file mode 100644 index dc034c3906..0000000000 --- a/doc/design/session.md +++ /dev/null @@ -1,72 +0,0 @@ -# Design Doc: Session - -## Abstract - -This design doc proposes to have an object called *Session* which -encapsulates the environment in which the computation graph is -executed. - -The session is able to distinguish running a graph locally or -remotely, using CPU only or using one or more GPUs. Different sessions -have different runtime environments such as [scopes](./scope.md) and -device contexts. - - -## Background - -A computation graph runs in an environment which contains states such -as the scope and device contexts. The current design has an implicit -global session on which `paddle.eval()` is executed. - -Since the user is not able to explicitly switch between runtime -environments such as the scope and the device contexts, the user -cannot run a topology in two independent environments. For example, in -reinforcement learning, the user may want to have a stale model for -inference and a fresh model for training, and only replace the stale -model with the fresh model periodically. Also, we have no concept that -can encapsulate a remote environment that could execute a computation -graph. - -We need a session concept to address above issues. - -## Session - -A session is an object that owns all runtime states such as scope, -reader OP's file handles, connection to a remote PaddlePaddle cluster, -etc. - -The session has two methods: `eval` and `close`. `eval` executes the -target OP in a given graph, and `close` closes the session and -releases all related resources: - -```Python -a = paddle.constant(1.0) -b = paddle.constant(2.0) -c = a + b -sess = paddle.session() -sess.eval(c) -sess.close() -``` - -### Remote Session - -Paddle Cloud will support user creating a remote session pointing to -the Paddle Cloud cluster. The user can send the computation graph to -be executed on the Paddle Cloud. In this way, the user can control a -cluster from her local computer: - -```Python -reader = paddle.reader.recordio("/pfs/home/peter/mnist-train-*") # data stored on Paddle Cloud -image = reader.column(0) -label = reader.column(1) -fc1 = paddle.op.fc(image, size=256, act="sigmoid") -fc2 = paddle.op.fc(fc1, size=10, act="softmax") -cost = paddle.op.cross_entropy(fc2, label) -opt = paddle.optimizer.sgd(cost) - -remote_config = ... # remote configuration such as endpoint, number of nodes and authentication. -sess = paddle.remoteSession(remote_config) -for i in range(1000): - sess.eval(opt) -sess.close() -``` From 88a8eedda17dead5471f4d9a64e291e49b522775 Mon Sep 17 00:00:00 2001 From: zchen0211 Date: Thu, 28 Sep 2017 14:36:38 -0700 Subject: [PATCH 033/170] scatter gather gpu gather scatter gpu --- paddle/operators/cond_op.cc | 6 +- paddle/operators/gather.cu.h | 84 ++++++++++++++++++ paddle/operators/gather.h | 35 ++++---- paddle/operators/gather_op.cc | 9 +- paddle/operators/gather_op.cu | 70 +++++++++++++++ paddle/operators/gather_op.h | 27 ++++-- paddle/operators/gather_test.cc | 2 +- paddle/operators/scatter.cu.h | 86 +++++++++++++++++++ paddle/operators/scatter.h | 45 ++++------ paddle/operators/scatter_op.cc | 7 +- paddle/operators/scatter_op.cu | 63 ++++++++++++++ paddle/operators/scatter_op.h | 12 ++- paddle/operators/scatter_test.cc | 2 +- .../v2/framework/tests/test_scatter_op.py | 4 +- 14 files changed, 375 insertions(+), 77 deletions(-) create mode 100644 paddle/operators/gather.cu.h create mode 100644 paddle/operators/gather_op.cu create mode 100644 paddle/operators/scatter.cu.h create mode 100644 paddle/operators/scatter_op.cu diff --git a/paddle/operators/cond_op.cc b/paddle/operators/cond_op.cc index aaffa6661f..157656786a 100644 --- a/paddle/operators/cond_op.cc +++ b/paddle/operators/cond_op.cc @@ -169,8 +169,8 @@ void CondOp::Run(const Scope& scope, tensor_child->Resize(dim); tensor_child->mutable_data(dim, platform::CPUPlace()); - Gather(dev_ctx.GetPlace(), tensor_parent, &index_tensors[i], - tensor_child); + CPUTGather(dev_ctx.GetPlace(), tensor_parent, &index_tensors[i], + tensor_child); } } @@ -194,7 +194,7 @@ void CondOp::Run(const Scope& scope, PADDLE_ENFORCE_NOT_NULL(v); LoDTensor* tensor_child = v->GetMutable(); - ScatterUpdate(dev_ctx.GetPlace(), tensor_child, &index_tensors[i], + ScatterAssign(dev_ctx.GetPlace(), tensor_child, &index_tensors[i], tensor_parent); } } diff --git a/paddle/operators/gather.cu.h b/paddle/operators/gather.cu.h new file mode 100644 index 0000000000..c96071e295 --- /dev/null +++ b/paddle/operators/gather.cu.h @@ -0,0 +1,84 @@ +/* Copyright (c) 2016 PaddlePaddle Authors All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once +#include "paddle/framework/tensor.h" +#include "paddle/platform/place.h" + +namespace paddle { +namespace operators { + +using framework::Tensor; +using platform::Place; + +#define CUDA_1D_KERNEL_LOOP(i, n) \ + for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < (n); \ + i += blockDim.x * gridDim.x) + +template +__global__ void GatherCUDAKernel(const T* params, const int* indices, T* output, + size_t index_size, size_t slice_size) { + CUDA_1D_KERNEL_LOOP(i, index_size * slice_size) { + int indices_i = i / slice_size; + int slice_i = i - indices_i * slice_size; // offset inside the slice + int gather_i = indices[indices_i]; + int params_i = gather_i * slice_size + slice_i; + *(output + i) = *(params + params_i); + } +} + +// Implementation of GPU copy: +template +struct GPUGather { + void operator()(const T* src, const int* index, const int slice_size, + const int index_size, T* output) { + int block = 512; + int n = slice_size * index_size; + int grid = (n + block - 1) / block; + GatherCUDAKernel<<>>(src, index, output, index_size, + slice_size); + } +}; + +/** + * A thin wrapper on gpu tensor + * Return a new tensor from source tensor, gathered according to index + * input[src]: type-T source Tensor + * input[index]: type-int index Tensor (1-D) + * return: output tensor + */ +template +void GPUTGather(const Place& place, const Tensor* src, const Tensor* index, + Tensor* output) { + PADDLE_ENFORCE(platform::is_gpu_place(place)); + // check index of shape 1-D + PADDLE_ENFORCE(index->dims().size() == 1); + int index_size = index->dims()[0]; + + auto src_dims = src->dims(); + framework::DDim output_dims(src_dims); + output_dims[0] = index_size; + + // slice size + int slice_size = 1; + for (int i = 1; i < src_dims.size(); ++i) slice_size *= src_dims[i]; + + // Gathering + GPUGather gather_functor; + gather_functor(src->data(), index->data(), slice_size, index_size, + output->data()); +} + +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/gather.h b/paddle/operators/gather.h index 92fb51ec17..a3db17bd3d 100644 --- a/paddle/operators/gather.h +++ b/paddle/operators/gather.h @@ -26,31 +26,31 @@ namespace operators { // Implementation of CPU copy template -void CPUGather(const T* src, const int* indices, const int slice_size, - const int index_size, T* output) { - const size_t slice_bytes = slice_size * sizeof(T); +struct CPUGather { + void operator()(const T* src, const int* indices, const int slice_size, + const int index_size, T* output) { + const size_t slice_bytes = slice_size * sizeof(T); - for (int i = 0; i < index_size; ++i) { - int index_ = indices[i]; - memcpy(output + i * slice_size, src + index_ * slice_size, slice_bytes); + for (int i = 0; i < index_size; ++i) { + int index_ = indices[i]; + memcpy(output + i * slice_size, src + index_ * slice_size, slice_bytes); + } } -} - -// Implementation of GPU copy: -template -void GPUGather(const T* src, const int* index, const int slice_size, - const int index_size, T* output); +}; /** + * A thin wrapper on cpu tensor * Return a new tensor from source tensor, gathered according to index * input[src]: type-T source Tensor * input[index]: type-int index Tensor (1-D) * return: output tensor */ template -void Gather(const platform::Place& place, const paddle::framework::Tensor* src, - const paddle::framework::Tensor* index, - paddle::framework::Tensor* output) { +void CPUTGather(const platform::Place& place, + const paddle::framework::Tensor* src, + const paddle::framework::Tensor* index, + paddle::framework::Tensor* output) { + PADDLE_ENFORCE(platform::is_cpu_place(place)); // check index of shape 1-D PADDLE_ENFORCE(index->dims().size() == 1); int index_size = index->dims()[0]; @@ -64,10 +64,9 @@ void Gather(const platform::Place& place, const paddle::framework::Tensor* src, for (int i = 1; i < src_dims.size(); ++i) slice_size *= src_dims[i]; // Gathering - if (platform::is_cpu_place(place)) { - CPUGather(src->data(), index->data(), slice_size, index_size, + CPUGather gather_functor; + gather_functor(src->data(), index->data(), slice_size, index_size, output->data()); - } } } // namespace operators diff --git a/paddle/operators/gather_op.cc b/paddle/operators/gather_op.cc index da22bd0c52..fe305337cb 100644 --- a/paddle/operators/gather_op.cc +++ b/paddle/operators/gather_op.cc @@ -31,6 +31,8 @@ class GatherOp : public framework::OperatorWithKernel { PADDLE_ENFORCE(ctx->HasOutput("Out"), "Output(Out) of GatherOp should not be null."); + auto index_dims = ctx->GetInputDim("Index"); + PADDLE_ENFORCE(index_dims.size() == 1); int batch_size = ctx->GetInputDim("Index")[0]; PADDLE_ENFORCE_GE(batch_size, 0, "Batch size must be >0"); framework::DDim output_dims(ctx->GetInputDim("X")); @@ -79,8 +81,5 @@ Out = X[Index] namespace ops = paddle::operators; REGISTER_OP(gather, ops::GatherOp, ops::GatherOpMaker, gather_grad, ops::GatherGradOp); -REGISTER_OP_CPU_KERNEL(gather, - ops::GatherOpKernel); -REGISTER_OP_CPU_KERNEL( - gather_grad, - ops::GatherGradientOpKernel); +REGISTER_OP_CPU_KERNEL(gather, ops::GatherOpKernel); +REGISTER_OP_CPU_KERNEL(gather_grad, ops::GatherGradientOpKernel); diff --git a/paddle/operators/gather_op.cu b/paddle/operators/gather_op.cu new file mode 100644 index 0000000000..f3ed692666 --- /dev/null +++ b/paddle/operators/gather_op.cu @@ -0,0 +1,70 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "gather.cu.h" +#include "paddle/framework/eigen.h" +#include "paddle/operators/gather_op.h" +#include "scatter.cu.h" + +namespace paddle { +namespace operators { + +// template +__global__ void print_arr(const float *params, const int N) { + CUDA_1D_KERNEL_LOOP(i, N) { printf("device: %d, %f\n", i, params[i]); } +} + +template +class GatherOpCUDAKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext &ctx) const override { + PADDLE_ENFORCE(platform::is_gpu_place(ctx.GetPlace()), + "This kernel only runs on GPU device."); + auto *x = ctx.Input("X"); + auto *index = ctx.Input("Index"); + auto *output = ctx.Output("Out"); + + output->mutable_data(ctx.GetPlace()); + + GPUTGather(ctx.GetPlace(), x, index, output); + } +}; + +template +class GatherGradOpCUDAKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext &ctx) const override { + PADDLE_ENFORCE(platform::is_gpu_place(ctx.GetPlace()), + "This kernel only runs on GPU device."); + LOG(INFO) << "Gather grad here"; + auto *Index = ctx.Input("Index"); + auto *dX = ctx.Output(framework::GradVarName("X")); + auto *dO = ctx.Input(framework::GradVarName("Out")); + auto *x = ctx.Input("X"); + + dX->mutable_data(ctx.GetPlace()); + auto dxt = framework::EigenVector::Flatten(*dX); + auto place = ctx.GetEigenDevice(); + dxt.device(place) = dxt.constant(static_cast(0)); + + GPUTScatter(ctx.GetPlace(), dO, Index, dX); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP_GPU_KERNEL(gather, ops::GatherOpCUDAKernel); +REGISTER_OP_GPU_KERNEL(gather_grad, ops::GatherGradOpCUDAKernel); diff --git a/paddle/operators/gather_op.h b/paddle/operators/gather_op.h index 073e566e8f..b80a4ab370 100644 --- a/paddle/operators/gather_op.h +++ b/paddle/operators/gather_op.h @@ -23,29 +23,40 @@ namespace operators { using Tensor = framework::Tensor; -template +template class GatherOpKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext &ctx) const override { - auto *X = ctx.Input("X"); - auto *Index = ctx.Input("Index"); - auto *Y = ctx.Output("Out"); + PADDLE_ENFORCE(platform::is_cpu_place(ctx.GetPlace()), + "This kernel only runs on CPU."); + + auto *x = ctx.Input("X"); + auto *index = ctx.Input("Index"); + auto *output = ctx.Output("Out"); + + output->mutable_data(ctx.GetPlace()); - Y->mutable_data(ctx.GetPlace()); - Gather(ctx.GetPlace(), X, Index, Y); + CPUTGather(ctx.GetPlace(), x, index, output); } }; -template +template class GatherGradientOpKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext &ctx) const override { + PADDLE_ENFORCE(platform::is_cpu_place(ctx.GetPlace()), + "This kernel only runs on CPU."); + auto *Index = ctx.Input("Index"); auto *dX = ctx.Output(framework::GradVarName("X")); auto *dO = ctx.Input(framework::GradVarName("Out")); dX->mutable_data(ctx.GetPlace()); - ScatterUpdate(ctx.GetPlace(), dO, Index, dX); + auto dxt = framework::EigenVector::Flatten(*dX); + auto place = ctx.GetEigenDevice(); + dxt.device(place) = dxt.constant(static_cast(0)); + + ScatterAssign(ctx.GetPlace(), dO, Index, dX); } }; diff --git a/paddle/operators/gather_test.cc b/paddle/operators/gather_test.cc index 0ae1e99452..ea06ae2847 100644 --- a/paddle/operators/gather_test.cc +++ b/paddle/operators/gather_test.cc @@ -41,7 +41,7 @@ TEST(Gather, GatherData) { int* p_output = output->mutable_data(make_ddim({2, 4}), CPUPlace()); - Gather(CPUPlace(), src, index, output); + CPUTGather(CPUPlace(), src, index, output); for (int i = 0; i < 4; ++i) EXPECT_EQ(p_output[i], i + 4); for (int i = 4; i < 8; ++i) EXPECT_EQ(p_output[i], i - 4); diff --git a/paddle/operators/scatter.cu.h b/paddle/operators/scatter.cu.h new file mode 100644 index 0000000000..82e5040305 --- /dev/null +++ b/paddle/operators/scatter.cu.h @@ -0,0 +1,86 @@ +/* Copyright (c) 2016 PaddlePaddle Authors All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once +#include "paddle/framework/tensor.h" +#include "paddle/platform/place.h" + +namespace paddle { +namespace operators { + +#define CUDA_1D_KERNEL_LOOP(i, n) \ + for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < (n); \ + i += blockDim.x * gridDim.x) + +template +__global__ void ScatterCUDAKernel(const T* params, const int* indices, + T* output, size_t index_size, + size_t slice_size) { + CUDA_1D_KERNEL_LOOP(i, index_size * slice_size) { + int indices_i = i / slice_size; + int slice_i = i - indices_i * slice_size; // offset inside the slice + int scatter_i = indices[indices_i]; + int out_i = scatter_i * slice_size + slice_i; + *(output + out_i) = *(params + i); + } +} + +// Implementation of GPU copy: +template +struct GPUScatterAssign { + void operator()(const T* src, const int* index, const int slice_size, + const int index_size, T* output) { + int block = 512; + int n = slice_size * index_size; + int grid = (n + block - 1) / block; + // printf("grid, block: %d %d\n", grid, block); + ScatterCUDAKernel<<>>(src, index, output, index_size, + slice_size); + } +}; + +/** + * A thin wrapper on gpu tensor + * Return a new updated tensor from source tensor, scatter-assigned according to + * index + * input[src]: type-T source Tensor + * input[index]: type-int index Tensor (1-D) + * return: output tensor + */ +template +void GPUTScatter(const platform::Place& place, + const paddle::framework::Tensor* src, + const paddle::framework::Tensor* index, + paddle::framework::Tensor* output) { + PADDLE_ENFORCE(platform::is_gpu_place(place)); + // check index of shape 1-D + PADDLE_ENFORCE(index->dims().size() == 1); + int index_size = index->dims()[0]; + + auto src_dims = src->dims(); + framework::DDim output_dims(src_dims); + output_dims[0] = index_size; + + // slice size + int slice_size = 1; + for (int i = 1; i < src_dims.size(); ++i) slice_size *= src_dims[i]; + + // Scatter Assign + GPUScatterAssign scatter_functor; + scatter_functor(src->data(), index->data(), slice_size, index_size, + output->data()); +} + +} // namespace operators +} // namespace paddle diff --git a/paddle/operators/scatter.h b/paddle/operators/scatter.h index 6b542675c2..670204b4dd 100644 --- a/paddle/operators/scatter.h +++ b/paddle/operators/scatter.h @@ -24,49 +24,33 @@ namespace paddle { namespace operators { using Tensor = framework::Tensor; -template -using EigenVector = framework::EigenVector; // Implementation of CPU copy template -void CPUScatterUpdate(const paddle::framework::Tensor* src, const int* index, - const size_t index_size, - paddle::framework::Tensor* output) { - paddle::framework::DDim output_dims = output->dims(); +void CPUScatterAssign(const T* src, const int* index, const int slice_size, + const int index_size, T* output) { + // paddle::framework::DDim output_dims = output->dims(); + const size_t slice_bytes = slice_size * sizeof(T); - for (size_t i = 0; i < index_size; ++i) { + for (int i = 0; i < index_size; ++i) { int index_ = index[i]; - - paddle::framework::Tensor src_ = *src; - paddle::framework::Tensor output_ = *output; - if (index_size > 1) src_ = src->Slice(i, i + 1); - if (output_dims[0] > 1) output_ = output->Slice(index_, index_ + 1); - - auto X = EigenVector::Flatten(src_); - auto Y = EigenVector::Flatten(output_); - - Y = X + Y; + memcpy(output + index_ * slice_size, src + i * slice_size, slice_bytes); } } -// Implementation of GPU scatter: -template -void GPUScatterUpdate(const T* src, const int* index, const int slice_size, - const int index_size, T* output); - /** * Return a updated tensor from source tensor, scattered according to index: - * dst[i] += src[index[i]] + * dst[i] = src[index[i]] * input[src]: type-T source Tensor * input[index]: type-int index Tensor (1-D) * return: output tensor */ template -void ScatterUpdate(const platform::Place& place, +void ScatterAssign(const platform::Place& place, const paddle::framework::Tensor* src, const paddle::framework::Tensor* index, paddle::framework::Tensor* output) { + PADDLE_ENFORCE(platform::is_cpu_place(place)); // check index of shape 1-D PADDLE_ENFORCE(index->dims().size() == 1); int index_size = index->dims()[0]; @@ -74,18 +58,19 @@ void ScatterUpdate(const platform::Place& place, auto src_dims = src->dims(); auto dst_dims = output->dims(); + const T* p_src = src->data(); + const int* p_index = index->data(); + T* p_output = output->data(); + // check src shape and dst shape should match for (int i = 1; i < src_dims.size(); i++) PADDLE_ENFORCE(src_dims[i] == dst_dims[i]); // slice size size_t slice_size = 1; - for (int i = 0; i < src_dims.size(); ++i) slice_size *= src_dims[i]; + for (int i = 1; i < src_dims.size(); ++i) slice_size *= src_dims[i]; - if (platform::is_cpu_place(place)) { - CPUScatterUpdate(src, index->data(), index_size, output); - } else { - } + CPUScatterAssign(p_src, p_index, slice_size, index_size, p_output); } } // namespace operators diff --git a/paddle/operators/scatter_op.cc b/paddle/operators/scatter_op.cc index cadd8841b6..d15ba15153 100644 --- a/paddle/operators/scatter_op.cc +++ b/paddle/operators/scatter_op.cc @@ -97,8 +97,5 @@ Out[Index] = Ref[Index] + Updates namespace ops = paddle::operators; REGISTER_OP(scatter, ops::ScatterOp, ops::ScatterOpMaker, scatter_grad, ops::ScatterGradOp); -REGISTER_OP_CPU_KERNEL(scatter, - ops::ScatterOpKernel); -REGISTER_OP_CPU_KERNEL( - scatter_grad, - ops::ScatterGradientOpKernel); +REGISTER_OP_CPU_KERNEL(scatter, ops::ScatterOpKernel); +REGISTER_OP_CPU_KERNEL(scatter_grad, ops::ScatterGradientOpKernel); diff --git a/paddle/operators/scatter_op.cu b/paddle/operators/scatter_op.cu new file mode 100644 index 0000000000..e27a926c6a --- /dev/null +++ b/paddle/operators/scatter_op.cu @@ -0,0 +1,63 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "gather.cu.h" +#include "paddle/operators/gather_op.h" +#include "scatter.cu.h" + +namespace paddle { +namespace operators { + +template +class ScatterOpCUDAKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext &ctx) const override { + PADDLE_ENFORCE(platform::is_gpu_place(ctx.GetPlace()), + "This kernel only runs on GPU device."); + auto *Ref = ctx.Input("Ref"); + auto *Index = ctx.Input("Index"); + auto *Updates = ctx.Input("Updates"); + auto *Out = ctx.Output("Out"); + + Out->ShareDataWith(*Ref); + + GPUTScatter(ctx.GetPlace(), Updates, Index, Out); + } +}; + +template +class ScatterGradOpCUDAKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext &ctx) const override { + PADDLE_ENFORCE(platform::is_gpu_place(ctx.GetPlace()), + "This kernel only runs on GPU device."); + auto *dRef = ctx.Output(framework::GradVarName("Ref")); + auto *dUpdates = ctx.Output(framework::GradVarName("Updates")); + auto *Index = ctx.Input("Index"); + auto *dOut = ctx.Input(framework::GradVarName("Out")); + + // In place gradient: dRef = dO + dRef->ShareDataWith(*dOut); + dUpdates->mutable_data(ctx.GetPlace()); + // Gradient by Gather: dUpdates = dO[Index] + GPUTGather(ctx.GetPlace(), dOut, Index, dUpdates); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP_GPU_KERNEL(scatter, ops::ScatterOpCUDAKernel); +REGISTER_OP_GPU_KERNEL(scatter_grad, ops::ScatterGradOpCUDAKernel); diff --git a/paddle/operators/scatter_op.h b/paddle/operators/scatter_op.h index a8eb54399a..74b2718f43 100644 --- a/paddle/operators/scatter_op.h +++ b/paddle/operators/scatter_op.h @@ -23,10 +23,12 @@ namespace operators { using Tensor = framework::Tensor; -template +template class ScatterOpKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext &ctx) const override { + PADDLE_ENFORCE(platform::is_cpu_place(ctx.GetPlace()), + "This kernel only runs on CPU."); auto *Ref = ctx.Input("Ref"); auto *Index = ctx.Input("Index"); auto *Updates = ctx.Input("Updates"); @@ -35,14 +37,16 @@ class ScatterOpKernel : public framework::OpKernel { // In place output: Out = Ref, Out[Index] += Updates Out->ShareDataWith(*Ref); // Apply ScatterUpdate: Out[index] += Updates[:] - ScatterUpdate(ctx.GetPlace(), Updates, Index, Out); + ScatterAssign(ctx.GetPlace(), Updates, Index, Out); } }; -template +template class ScatterGradientOpKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext &ctx) const override { + PADDLE_ENFORCE(platform::is_cpu_place(ctx.GetPlace()), + "This kernel only runs on CPU."); auto *dRef = ctx.Output(framework::GradVarName("Ref")); auto *dUpdates = ctx.Output(framework::GradVarName("Updates")); auto *Index = ctx.Input("Index"); @@ -52,7 +56,7 @@ class ScatterGradientOpKernel : public framework::OpKernel { dRef->ShareDataWith(*dOut); dUpdates->mutable_data(ctx.GetPlace()); // Gradient by Gather: dUpdates += dO[Index] - Gather(ctx.GetPlace(), dOut, Index, dUpdates); + CPUTGather(ctx.GetPlace(), dOut, Index, dUpdates); } }; diff --git a/paddle/operators/scatter_test.cc b/paddle/operators/scatter_test.cc index 26fdaff146..bace6419d0 100644 --- a/paddle/operators/scatter_test.cc +++ b/paddle/operators/scatter_test.cc @@ -40,7 +40,7 @@ TEST(scatter, ScatterUpdate) { float* p_output = output->mutable_data(make_ddim({4, 4}), CPUPlace()); - ScatterUpdate(CPUPlace(), src, index, output); + ScatterAssign(CPUPlace(), src, index, output); for (size_t i = 0; i < 4; ++i) EXPECT_EQ(p_output[i], float(0)); for (size_t i = 0; i < 4; ++i) EXPECT_EQ(output->data()[i], float(0)); diff --git a/python/paddle/v2/framework/tests/test_scatter_op.py b/python/paddle/v2/framework/tests/test_scatter_op.py index 33c73c5263..1032269d5d 100644 --- a/python/paddle/v2/framework/tests/test_scatter_op.py +++ b/python/paddle/v2/framework/tests/test_scatter_op.py @@ -10,7 +10,7 @@ class TestScatterOp(OpTest): index_np = np.array([1, 2]).astype("int32") updates_np = np.random.random((2, 3)).astype("float32") output_np = np.copy(ref_np) - output_np[index_np] += updates_np + output_np[index_np] = updates_np self.inputs = {'Ref': ref_np, 'Index': index_np, 'Updates': updates_np} self.outputs = {'Out': output_np} @@ -18,7 +18,7 @@ class TestScatterOp(OpTest): self.check_output() def test_check_grad(self): - self.check_grad(['Updates', 'Ref'], 'Out', in_place=True) + self.check_grad(['Updates'], 'Out', in_place=True) if __name__ == "__main__": From 757c76b83f3701c29efc88c546ec90a18952f98a Mon Sep 17 00:00:00 2001 From: Helin Wang Date: Thu, 28 Sep 2017 15:07:17 -0700 Subject: [PATCH 034/170] update according to comments --- doc/design/refactor/session.md | 74 +++++++++++++++++++++------------- 1 file changed, 47 insertions(+), 27 deletions(-) diff --git a/doc/design/refactor/session.md b/doc/design/refactor/session.md index 5f58148f01..9a7451ece5 100644 --- a/doc/design/refactor/session.md +++ b/doc/design/refactor/session.md @@ -5,17 +5,17 @@ The *session* object encapsulates the environment in which the computation graph is executed. -We will have *local* session and *remote* session, they offer the +We will have the *local* session and *remote* session, they offer the same [interface](#interface). The local session encapsulates the local runtime environment and the remote session encapsulates the cluster -runtime envrionment. +runtime environment. -The local runtime envrionment contains: +The local runtime environment contains: 1. computation devices (i.e., CPU, GPU) handles, and 1. the [scope](../scope.md) which holds all variables. -The remote runtime envrionment contains: +The remote runtime environment contains: 1. computation devices (i.e., CPU and GPU on node 0, 1) in a cluster, and @@ -29,12 +29,12 @@ remote computation resource in a cluster from his local computer. ## Background -The current design has an implicit global session on which +The current design has an implicit global session in which `paddle.eval()` is executed. The pain point is: Since the user is not able to explicitly switch between runtime -environments such as the scope and the device contexts, the user -cannot run a topology in two independent environments. +environments, the user cannot run a topology in two independent +environments. For example, in reinforcement learning, the user may want to have a stale model for inference and a fresh model for training, and only @@ -49,12 +49,12 @@ We need the session object to address above issues. ## Session A session is an object that owns the runtime environment. All -computations are executed through `session.eval`. +computations are executed through `session.eval()`. ### Interface -``` +```python eval( targets, feed_dict=None, @@ -64,37 +64,57 @@ eval( Evaluates the target Operations or Variables in `targets`. - *targets*: the evaluation targets. Can be a single Operation or - Variable, or a list with the Operations or Variables as elements. + Variable, or a list with the Operations or Variables as + elements. The value returned by `eval()` has the same shape as the + `target` argument. + + The PaddlePaddle program is represented by + the [ProgramDesc](../design/program.md), `eval()` will infer the + ProgramDesc from the given targets and run the PaddlePaddle + program. Please + see + [this graph](./distributed_architecture.md#local-training-architecture) for + the detailed illustration for the local session + and + [this graph](./distributed_architecture.md#distributed-training-architecture) for + the detailed illustration for the remote session. + +- *feed_dict*: a dictionary that contains the tensors which override + the edges of the computation graph. - The value returned by `eval()` has the same shape as the `target` - argument. + feed_dict not only can provide the input data, it can override any + OP's input as well: - The computation graph is implicitly inferred from the targets. + ```python + a = pd.constant(1.0, name="a") + b = pd.constant(2.0) + c = pd.mul(a,b) + sess.eval(targets=c, feed_dict={"a":3.0}) # returns 6.0 + ``` -- *feed_dict*: a dictionary that contains the tensors which overrides - the edges of the computation graph. - -``` +```python close() ``` -Closes the session. Calling this method releases the scope. +Closes the session and releases the scope that the session owns. ### Create a Local Session -``` +```python session( - gpu_ids=None + devices=None ) ``` Creates a new session. One session owns one scope, so creating multiple sessions will create different scopes. -- *gpu_ids*: a single `int` or a list of `int` of the GPU IDs to be - used as the computation devices. If not specified, all avaiable GPUs - will be used. +- *devices*: a single `string` or a list of `string` of device names, + the corresponding devices will be the computation devices for + `eval()`. If not specified, all available devices (e.g., all GPUs) + will be used. The user doesn't need to specify the CPU device since + it will be always used. #### Example @@ -103,14 +123,14 @@ multiple sessions will create different scopes. a = paddle.constant(1.0) b = paddle.constant(2.0) c = a + b -sess = paddle.session(gpu_ids=[0,1]) +sess = paddle.session(devices=["gpu:0", "gpu:1", "fpga:0"]) sess.eval(c) sess.close() ``` ### Create a Remote Session -``` +```python create_cloud_job( name, num_trainer, @@ -125,7 +145,7 @@ create_cloud_job( Creates a Paddle Cloud job. Fails if the job name exists. -``` +```python get_cloud_job( name ) @@ -133,7 +153,7 @@ get_cloud_job( Gets a Paddle Cloud job. -``` +```python remote_session( job ) From b851515b16d179f35410836a17f855b9b6a9c268 Mon Sep 17 00:00:00 2001 From: zchen0211 Date: Thu, 28 Sep 2017 15:41:20 -0700 Subject: [PATCH 035/170] merge new op grammar --- paddle/operators/gather_op.cu | 9 ++------- paddle/operators/scatter_op.cu | 4 ++-- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/paddle/operators/gather_op.cu b/paddle/operators/gather_op.cu index f3ed692666..f7533cdd64 100644 --- a/paddle/operators/gather_op.cu +++ b/paddle/operators/gather_op.cu @@ -20,13 +20,8 @@ namespace paddle { namespace operators { -// template -__global__ void print_arr(const float *params, const int N) { - CUDA_1D_KERNEL_LOOP(i, N) { printf("device: %d, %f\n", i, params[i]); } -} - template -class GatherOpCUDAKernel : public framework::OpKernel { +class GatherOpCUDAKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext &ctx) const override { PADDLE_ENFORCE(platform::is_gpu_place(ctx.GetPlace()), @@ -42,7 +37,7 @@ class GatherOpCUDAKernel : public framework::OpKernel { }; template -class GatherGradOpCUDAKernel : public framework::OpKernel { +class GatherGradOpCUDAKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext &ctx) const override { PADDLE_ENFORCE(platform::is_gpu_place(ctx.GetPlace()), diff --git a/paddle/operators/scatter_op.cu b/paddle/operators/scatter_op.cu index e27a926c6a..89d23945e0 100644 --- a/paddle/operators/scatter_op.cu +++ b/paddle/operators/scatter_op.cu @@ -20,7 +20,7 @@ namespace paddle { namespace operators { template -class ScatterOpCUDAKernel : public framework::OpKernel { +class ScatterOpCUDAKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext &ctx) const override { PADDLE_ENFORCE(platform::is_gpu_place(ctx.GetPlace()), @@ -37,7 +37,7 @@ class ScatterOpCUDAKernel : public framework::OpKernel { }; template -class ScatterGradOpCUDAKernel : public framework::OpKernel { +class ScatterGradOpCUDAKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext &ctx) const override { PADDLE_ENFORCE(platform::is_gpu_place(ctx.GetPlace()), From 708cff32b1f8b9cf6da2c1a4a5d8458fea6eea0a Mon Sep 17 00:00:00 2001 From: Kavya Srinet Date: Thu, 28 Sep 2017 16:13:06 -0700 Subject: [PATCH 036/170] Few typos in refactorization file --- doc/design/refactorization.md | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/doc/design/refactorization.md b/doc/design/refactorization.md index a07675b3e0..715f209d3f 100644 --- a/doc/design/refactorization.md +++ b/doc/design/refactorization.md @@ -137,19 +137,18 @@ Compile Time -> IR -> Runtime * `Eigen::Tensor` contains basic math and element-wise functions. * Note that `Eigen::Tensor` has broadcast implementation. * Limit the number of `tensor.device(dev) = ` in your code. -* `thrust::tranform` and `std::transform`. - * `thrust` has the same API as C++ standard library. Using `transform`, one can quickly implement customized elementwise kernels. +* `thrust::transform` and `std::transform`. + * `thrust` has the same API as C++ standard library. Using `transform`, one can quickly implement customized element-wise kernels. * `thrust` also has more complex APIs, like `scan`, `reduce`, `reduce_by_key`. * Hand-writing `GPUKernel` and `CPU` code * Do not write in header (`.h`) files. CPU Kernel should be in cpp source (`.cc`) and GPU kernels should be in cuda (`.cu`) files. (GCC cannot compile GPU code.) --- # Operator Registration -## Why registration is necessary? +## Why is registration necessary? We need a method to build mappings between Op type names and Op classes. ## How is registration implemented? - Maintaining a map, whose key is the type name and the value is the corresponding Op constructor. --- @@ -170,7 +169,7 @@ Maintaining a map, whose key is the type name and the value is the corresponding # Related Concepts ### Op_Maker -It's constructor takes `proto` and `checker`. They are compeleted during Op_Maker's construction. ([ScaleOpMaker](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/operators/scale_op.cc#L37)) +It's constructor takes `proto` and `checker`. They are completed during Op_Maker's construction. ([ScaleOpMaker](https://github.com/PaddlePaddle/Paddle/blob/develop/paddle/operators/scale_op.cc#L37)) ### Register Macros ```cpp @@ -200,7 +199,7 @@ Make sure the registration process is executed and linked. --- # Backward Module (2/2) ### Build Backward Network -- **Input**: graph of forwarding operators +- **Input**: graph of forward operators - **Output**: graph of backward operators - **Corner cases in construction** - Shared Variables => insert an `Add` operator to combine gradients @@ -224,7 +223,7 @@ Make sure the registration process is executed and linked. --- # Block (in design) -## the difference with original RNNOp +## the difference between original RNNOp and Block - As an operator is more intuitive than `RNNOp`, - Offers a new interface `Eval(targets)` to deduce the minimal block to `Run`, - Fits the compile-time/ runtime separation design paradigm. From 337b7ebe7787a522f679344509523f02047c67aa Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Thu, 28 Sep 2017 16:56:51 -0700 Subject: [PATCH 037/170] Unify Activation functions and simplify register code --- paddle/operators/activation_op.cc | 83 ++------ paddle/operators/activation_op.cu | 94 +-------- paddle/operators/activation_op.h | 324 +++++++++++++++--------------- 3 files changed, 190 insertions(+), 311 deletions(-) diff --git a/paddle/operators/activation_op.cc b/paddle/operators/activation_op.cc index f77e1c572e..33826fc206 100644 --- a/paddle/operators/activation_op.cc +++ b/paddle/operators/activation_op.cc @@ -195,111 +195,54 @@ class STanhOpMaker : public framework::OpProtoAndCheckerMaker { } // namespace paddle namespace ops = paddle::operators; + REGISTER_OP(sigmoid, ops::ActivationOp, ops::SigmoidOpMaker, sigmoid_grad, ops::ActivationOpGrad); -REGISTER_OP_CPU_KERNEL(sigmoid, - ops::ActivationKernel>); -REGISTER_OP_CPU_KERNEL( - sigmoid_grad, ops::ActivationGradKernel>); REGISTER_OP(exp, ops::ActivationOp, ops::ExpOpMaker, exp_grad, ops::ActivationOpGrad); -REGISTER_OP_CPU_KERNEL( - exp, - ops::ActivationKernel); -REGISTER_OP_CPU_KERNEL(exp_grad, - ops::ActivationGradKernel); REGISTER_OP(relu, ops::ActivationOp, ops::ReluOpMaker, relu_grad, ops::ActivationOpGrad); -REGISTER_OP_CPU_KERNEL(relu, - ops::ActivationKernel>); -REGISTER_OP_CPU_KERNEL( - relu_grad, ops::ActivationGradKernel>); REGISTER_OP(tanh, ops::ActivationOp, ops::TanhOpMaker, tanh_grad, ops::ActivationOpGrad); -REGISTER_OP_CPU_KERNEL( - tanh, - ops::ActivationKernel); -REGISTER_OP_CPU_KERNEL( - tanh_grad, ops::ActivationGradKernel>); REGISTER_OP(sqrt, ops::ActivationOp, ops::SqrtOpMaker, sqrt_grad, ops::ActivationOpGrad); -REGISTER_OP_CPU_KERNEL( - sqrt, - ops::ActivationKernel); -REGISTER_OP_CPU_KERNEL( - sqrt_grad, ops::ActivationGradKernel>); REGISTER_OP(abs, ops::ActivationOp, ops::AbsOpMaker, abs_grad, ops::ActivationOpGrad); -REGISTER_OP_CPU_KERNEL( - abs, - ops::ActivationKernel); -REGISTER_OP_CPU_KERNEL(abs_grad, - ops::ActivationGradKernel); REGISTER_OP(reciprocal, ops::ActivationOp, ops::ReciprocalOpMaker, reciprocal_grad, ops::ActivationOpGrad); -REGISTER_OP_CPU_KERNEL(reciprocal, - ops::ActivationKernel>); -REGISTER_OP_CPU_KERNEL( - reciprocal_grad, - ops::ActivationGradKernel>); REGISTER_OP(log, ops::ActivationOp, ops::LogOpMaker, log_grad, ops::ActivationOpGrad); -REGISTER_OP_CPU_KERNEL( - log, - ops::ActivationKernel); -REGISTER_OP_CPU_KERNEL( - log_grad, ops::ActivationGradKernel>); REGISTER_OP(square, ops::ActivationOp, ops::SquareOpMaker, square_grad, ops::ActivationOpGrad); -REGISTER_OP_CPU_KERNEL(square, - ops::ActivationKernel); -REGISTER_OP_CPU_KERNEL( - square_grad, ops::ActivationGradKernel>); REGISTER_OP(brelu, ops::ActivationOp, ops::BReluOpMaker, brelu_grad, ops::ActivationOpGrad); -REGISTER_OP_CPU_KERNEL(brelu, - ops::BReluKernel); -REGISTER_OP_CPU_KERNEL(brelu_grad, - ops::BReluGradKernel); REGISTER_OP(soft_relu, ops::ActivationOp, ops::SoftReluOpMaker, soft_relu_grad, ops::ActivationOpGrad); -REGISTER_OP_CPU_KERNEL(soft_relu, - ops::SoftReluKernel); -REGISTER_OP_CPU_KERNEL( - soft_relu_grad, ops::SoftReluGradKernel); REGISTER_OP(pow, ops::ActivationOp, ops::PowOpMaker, pow_grad, ops::ActivationOpGrad); -REGISTER_OP_CPU_KERNEL(pow, ops::PowKernel); -REGISTER_OP_CPU_KERNEL(pow_grad, - ops::PowGradKernel); REGISTER_OP(stanh, ops::ActivationOp, ops::STanhOpMaker, stanh_grad, ops::ActivationOpGrad); -REGISTER_OP_CPU_KERNEL(stanh, - ops::STanhKernel); -REGISTER_OP_CPU_KERNEL(stanh_grad, - ops::STanhGradKernel); + +#define REGISTER_ACTIVATION_CPU_KERNEL(act_type, functor, grad_functor) \ + REGISTER_OP_CPU_KERNEL( \ + act_type, \ + paddle::operators::ActivationKernel>); \ + REGISTER_OP_CPU_KERNEL(act_type##_grad, \ + paddle::operators::ActivationGradKernel< \ + paddle::platform::CPUPlace, \ + paddle::operators::grad_functor>); + +FOR_EACH_KERNEL_FUNCTOR(REGISTER_ACTIVATION_CPU_KERNEL); diff --git a/paddle/operators/activation_op.cu b/paddle/operators/activation_op.cu index feed1302b2..93e9f1c694 100644 --- a/paddle/operators/activation_op.cu +++ b/paddle/operators/activation_op.cu @@ -15,86 +15,14 @@ #define EIGEN_USE_GPU #include "paddle/operators/activation_op.h" -namespace ops = paddle::operators; - -REGISTER_OP_GPU_KERNEL(sigmoid, - ops::ActivationKernel>); -REGISTER_OP_GPU_KERNEL( - sigmoid_grad, ops::ActivationGradKernel>); - -REGISTER_OP_GPU_KERNEL( - exp, - ops::ActivationKernel); -REGISTER_OP_GPU_KERNEL(exp_grad, - ops::ActivationGradKernel); -REGISTER_OP_GPU_KERNEL(relu, - ops::ActivationKernel>); -REGISTER_OP_GPU_KERNEL( - relu_grad, ops::ActivationGradKernel>); - -REGISTER_OP_GPU_KERNEL( - tanh, - ops::ActivationKernel); -REGISTER_OP_GPU_KERNEL( - tanh_grad, ops::ActivationGradKernel>); - -REGISTER_OP_GPU_KERNEL( - sqrt, - ops::ActivationKernel); -REGISTER_OP_GPU_KERNEL( - sqrt_grad, ops::ActivationGradKernel>); - -REGISTER_OP_GPU_KERNEL( - abs, - ops::ActivationKernel); -REGISTER_OP_GPU_KERNEL(abs_grad, - ops::ActivationGradKernel); - -REGISTER_OP_GPU_KERNEL(reciprocal, - ops::ActivationKernel>); -REGISTER_OP_GPU_KERNEL( - reciprocal_grad, - ops::ActivationGradKernel>); - -REGISTER_OP_GPU_KERNEL( - log, - ops::ActivationKernel); -REGISTER_OP_GPU_KERNEL( - log_grad, ops::ActivationGradKernel>); - -REGISTER_OP_GPU_KERNEL(square, - ops::ActivationKernel); -REGISTER_OP_GPU_KERNEL( - square_grad, ops::ActivationGradKernel>); - -REGISTER_OP_GPU_KERNEL(brelu, - ops::BReluKernel); -REGISTER_OP_GPU_KERNEL(brelu_grad, - ops::BReluGradKernel); - -REGISTER_OP_GPU_KERNEL(soft_relu, - ops::SoftReluKernel); -REGISTER_OP_GPU_KERNEL( - soft_relu_grad, ops::SoftReluGradKernel); - -REGISTER_OP_GPU_KERNEL(pow, ops::PowKernel); -REGISTER_OP_GPU_KERNEL(pow_grad, - ops::PowGradKernel); - -REGISTER_OP_GPU_KERNEL(stanh, - ops::STanhKernel); -REGISTER_OP_GPU_KERNEL(stanh_grad, - ops::STanhGradKernel); +#define REGISTER_ACTIVATION_GPU_KERNEL(act_type, functor, grad_functor) \ + REGISTER_OP_GPU_KERNEL( \ + act_type, \ + paddle::operators::ActivationKernel>); \ + REGISTER_OP_GPU_KERNEL(act_type##_grad, \ + paddle::operators::ActivationGradKernel< \ + paddle::platform::GPUPlace, \ + paddle::operators::grad_functor>); + +FOR_EACH_KERNEL_FUNCTOR(REGISTER_ACTIVATION_GPU_KERNEL); diff --git a/paddle/operators/activation_op.h b/paddle/operators/activation_op.h index e400992ae2..d25bb204ff 100644 --- a/paddle/operators/activation_op.h +++ b/paddle/operators/activation_op.h @@ -19,9 +19,12 @@ namespace paddle { namespace operators { -template -class ActivationKernel : public framework::OpKernel { +template +class ActivationKernel + : public framework::OpKernel { public: + using T = typename Functor::ELEMENT_TYPE; + void Compute(const framework::ExecutionContext& context) const override { auto* X = context.Input("X"); auto* Y = context.Output("Y"); @@ -31,13 +34,20 @@ class ActivationKernel : public framework::OpKernel { auto y = framework::EigenVector::Flatten(*Y); auto place = context.GetEigenDevice(); Functor functor; + + auto attrs = functor.GetAttrs(); + for (auto& attr : attrs) { + *attr.second = context.Attr(attr.first); + } functor(place, x, y); } }; -template -class ActivationGradKernel : public framework::OpKernel { +template +class ActivationGradKernel + : public framework::OpKernel { public: + using T = typename Functor::ELEMENT_TYPE; void Compute(const framework::ExecutionContext& context) const override { auto* X = context.Input("X"); auto* Y = context.Input("Y"); @@ -51,303 +61,301 @@ class ActivationGradKernel : public framework::OpKernel { auto dx = framework::EigenVector::Flatten(*dX); auto place = context.GetEigenDevice(); Functor functor; + auto attrs = functor.GetAttrs(); + for (auto& attr : attrs) { + *attr.second = context.Attr(attr.first); + } functor(place, x, y, dy, dx); } }; +template +struct BaseActivationFunctor { + using ELEMENT_TYPE = T; + + using AttrPair = std::vector>; + + AttrPair GetAttrs() { return AttrPair(); } +}; + // sigmoid(x) = 1 / (1 + exp(-x)) template -struct SigmoidFunctor { +struct SigmoidFunctor : public BaseActivationFunctor { template - void operator()(Device d, X x, Y y) { + void operator()(Device d, X x, Y y) const { y.device(d) = static_cast(1) / (static_cast(1) + (-x).exp()); } }; template -struct SigmoidGradFunctor { +struct SigmoidGradFunctor : public BaseActivationFunctor { template - void operator()(Device d, X x, Y y, dY dy, dX dx) { + void operator()(Device d, X x, Y y, dY dy, dX dx) const { dx.device(d) = dy * y * (static_cast(1) - y); } }; // exp(x) = e^x -struct ExpFunctor { +template +struct ExpFunctor : public BaseActivationFunctor { template - void operator()(Device d, X x, Y y) { + void operator()(Device d, X x, Y y) const { y.device(d) = x.exp(); } }; -struct ExpGradFunctor { +template +struct ExpGradFunctor : public BaseActivationFunctor { template - void operator()(Device d, X x, Y y, dY dy, dX dx) { + void operator()(Device d, X x, Y y, dY dy, dX dx) const { dx.device(d) = dy * y; } }; // relu(x) = max(x, 0) template -struct ReluFunctor { +struct ReluFunctor : public BaseActivationFunctor { template - void operator()(Device d, X x, Y y) { + void operator()(Device d, X x, Y y) const { y.device(d) = x.cwiseMax(static_cast(0)); } }; template -struct ReluGradFunctor { +struct ReluGradFunctor : public BaseActivationFunctor { template - void operator()(Device d, X x, Y y, dY dy, dX dx) { + void operator()(Device d, X x, Y y, dY dy, dX dx) const { dx.device(d) = dy * (x > static_cast(0)).template cast(); } }; // tanh(x) = (exp(x) - exp(-x)) / (exp(x) + exp(-x)) -struct TanhFunctor { +template +struct TanhFunctor : public BaseActivationFunctor { template - void operator()(Device d, X x, Y y) { + void operator()(Device d, X x, Y y) const { y.device(d) = x.tanh(); } }; template -struct TanhGradFunctor { +struct TanhGradFunctor : public BaseActivationFunctor { template - void operator()(Device d, X x, Y y, dY dy, dX dx) { + void operator()(Device d, X x, Y y, dY dy, dX dx) const { dx.device(d) = dy * (static_cast(1) - y * y); } }; // sqrt(x) = x^(1/2) -struct SqrtFunctor { +template +struct SqrtFunctor : public BaseActivationFunctor { template - void operator()(Device d, X x, Y y) { + void operator()(Device d, X x, Y y) const { y.device(d) = x.sqrt(); } }; template -struct SqrtGradFunctor { +struct SqrtGradFunctor : public BaseActivationFunctor { template - void operator()(Device d, X x, Y y, dY dy, dX dx) { + void operator()(Device d, X x, Y y, dY dy, dX dx) const { const Y y_conj = Eigen::numext::conj(y); dx.device(d) = static_cast(0.5) * dy / y_conj; } }; // abs(x) = |x| -struct AbsFunctor { +template +struct AbsFunctor : public BaseActivationFunctor { template - void operator()(Device d, X x, Y y) { + void operator()(Device d, X x, Y y) const { y.device(d) = x.abs(); } }; -struct AbsGradFunctor { +template +struct AbsGradFunctor : public BaseActivationFunctor { template - void operator()(Device d, X x, Y y, dY dy, dX dx) { + void operator()(Device d, X x, Y y, dY dy, dX dx) const { dx.device(d) = dy * x.sign(); } }; // reciprocal(x) = 1 / x template -struct ReciprocalFunctor { +struct ReciprocalFunctor : public BaseActivationFunctor { template - void operator()(Device d, X x, Y y) { + void operator()(Device d, X x, Y y) const { y.device(d) = static_cast(1) / x; } }; template -struct ReciprocalGradFunctor { +struct ReciprocalGradFunctor : public BaseActivationFunctor { template - void operator()(Device d, X x, Y y, dY dy, dX dx) { + void operator()(Device d, X x, Y y, dY dy, dX dx) const { dx.device(d) = dy * static_cast(-1) * y * y; } }; // log(x) = natural logarithm of x -struct LogFunctor { +template +struct LogFunctor : public BaseActivationFunctor { template - void operator()(Device d, X x, Y y) { + void operator()(Device d, X x, Y y) const { y.device(d) = x.log(); } }; template -struct LogGradFunctor { +struct LogGradFunctor : public BaseActivationFunctor { template - void operator()(Device d, X x, Y y, dY dy, dX dx) { + void operator()(Device d, X x, Y y, dY dy, dX dx) const { dx.device(d) = dy * (static_cast(1) / x); } }; // square(x) = x^2 -struct SquareFunctor { +template +struct SquareFunctor : public BaseActivationFunctor { template - void operator()(Device d, X x, Y y) { + void operator()(Device d, X x, Y y) const { y.device(d) = x.square(); } }; template -struct SquareGradFunctor { +struct SquareGradFunctor : public BaseActivationFunctor { template - void operator()(Device d, X x, Y y, dY dy, dX dx) { + void operator()(Device d, X x, Y y, dY dy, dX dx) const { dx.device(d) = dy * static_cast(2) * x; } }; -template -class BReluKernel : public framework::OpKernel { - public: - void Compute(const framework::ExecutionContext& context) const override { - auto* X = context.Input("X"); - auto* Y = context.Output("Y"); - auto t_min = static_cast(context.Attr("t_min")); - auto t_max = static_cast(context.Attr("t_max")); - Y->mutable_data(context.GetPlace()); +template +struct BReluFunctor : public BaseActivationFunctor { + float t_min; + float t_max; + + // NOTE: Explicit hides the `BaseActivationFunctor::GetAttrs` + // not polymorphism for speed. + typename BaseActivationFunctor::AttrPair GetAttrs() { + return {{"t_min", &t_min}, {"t_max", &t_max}}; + } - auto x = framework::EigenVector::Flatten(*X); - auto y = framework::EigenVector::Flatten(*Y); - auto place = context.GetEigenDevice(); - y.device(place) = x.cwiseMax(t_min).cwiseMin(t_max); + template + void operator()(Device d, X x, Y y) const { + y.device(d) = x.cwiseMax(t_min).cwiseMin(t_max); } }; -template -class BReluGradKernel : public framework::OpKernel { - public: - void Compute(const framework::ExecutionContext& context) const override { - auto* X = context.Input("X"); - auto* dY = context.Input(framework::GradVarName("Y")); - auto* dX = context.Output(framework::GradVarName("X")); - auto t_min = static_cast(context.Attr("t_min")); - auto t_max = static_cast(context.Attr("t_max")); - dX->mutable_data(context.GetPlace()); - - auto dy = framework::EigenVector::Flatten(*dY); - auto x = framework::EigenVector::Flatten(*X); - auto dx = framework::EigenVector::Flatten(*dX); - auto place = context.GetEigenDevice(); - - dx.device(place) = dy * ((x > t_min) * (x < t_max)).template cast(); +template +struct BReluGradFunctor : public BaseActivationFunctor { + float t_min; + float t_max; + typename BaseActivationFunctor::AttrPair GetAttrs() { + return {{"t_min", &t_min}, {"t_max", &t_max}}; + } + template + void operator()(Device d, X x, Y y, dY dy, dX dx) const { + dx.device(d) = dy * ((x > t_min) * (x < t_max)).template cast(); } }; -template -class SoftReluKernel : public framework::OpKernel { - public: - void Compute(const framework::ExecutionContext& context) const override { - auto* X = context.Input("X"); - auto* Y = context.Output("Y"); - auto threshold = static_cast(context.Attr("threshold")); - Y->mutable_data(context.GetPlace()); +template +struct SoftReluFunctor : public BaseActivationFunctor { + float threshold; + typename BaseActivationFunctor::AttrPair GetAttrs() { + return {{"threshold", &threshold}}; + } - auto x = framework::EigenVector::Flatten(*X); - auto y = framework::EigenVector::Flatten(*Y); - auto place = context.GetEigenDevice(); - auto temp = x.cwiseMax(-threshold).cwiseMin(threshold).eval(); - y.device(place) = (static_cast(1) + temp.exp()).log(); + template + void operator()(Device d, X x, Y y) const { + auto temp = x.cwiseMax(-threshold).cwiseMin(threshold); + y.device(d) = (static_cast(1) + temp.exp()).log(); } }; -template -class SoftReluGradKernel : public framework::OpKernel { - public: - void Compute(const framework::ExecutionContext& context) const override { - auto* X = context.Input("X"); - auto* Y = context.Input("Y"); - auto* dY = context.Input(framework::GradVarName("Y")); - auto* dX = context.Output(framework::GradVarName("X")); - auto threshold = static_cast(context.Attr("threshold")); - dX->mutable_data(context.GetPlace()); - - auto x = framework::EigenVector::Flatten(*X); - auto y = framework::EigenVector::Flatten(*Y); - auto dy = framework::EigenVector::Flatten(*dY); - auto dx = framework::EigenVector::Flatten(*dX); - auto place = context.GetEigenDevice(); +template +struct SoftReluGradFunctor : public BaseActivationFunctor { + float threshold; + typename BaseActivationFunctor::AttrPair GetAttrs() { + return {{"threshold", &threshold}}; + } + template + void operator()(Device d, X x, Y y, dY dy, dX dx) const { auto temp = ((x > -threshold) * (x < threshold)).template cast().eval(); - dx.device(place) = dy * (static_cast(1) - (-y).exp()) * temp; + dx.device(d) = dy * (static_cast(1) - (-y).exp()) * temp; } }; -template -class PowKernel : public framework::OpKernel { - public: - void Compute(const framework::ExecutionContext& context) const override { - auto* X = context.Input("X"); - auto* Y = context.Output("Y"); - auto factor = static_cast(context.Attr("factor")); - Y->mutable_data(context.GetPlace()); - - auto x = framework::EigenVector::Flatten(*X); - auto y = framework::EigenVector::Flatten(*Y); - auto place = context.GetEigenDevice(); - y.device(place) = x.pow(factor); +template +struct PowFunctor : public BaseActivationFunctor { + float factor; + typename BaseActivationFunctor::AttrPair GetAttrs() { + return {{"factor", &factor}}; + } + template + void operator()(Device d, X x, Y y) const { + y.device(d) = x.pow(factor); } }; -template -class PowGradKernel : public framework::OpKernel { - public: - void Compute(const framework::ExecutionContext& context) const override { - auto* X = context.Input("X"); - auto* dY = context.Input(framework::GradVarName("Y")); - auto* dX = context.Output(framework::GradVarName("X")); - auto factor = static_cast(context.Attr("factor")); - dX->mutable_data(context.GetPlace()); - - auto dy = framework::EigenVector::Flatten(*dY); - auto x = framework::EigenVector::Flatten(*X); - auto dx = framework::EigenVector::Flatten(*dX); - auto place = context.GetEigenDevice(); - - dx.device(place) = dy * factor * x.pow(factor - static_cast(1)); +template +struct PowGradFunctor : public BaseActivationFunctor { + float factor; + typename BaseActivationFunctor::AttrPair GetAttrs() { + return {{"factor", &factor}}; + } + template + void operator()(Device d, X x, Y y, dY dy, dX dx) const { + dx.device(d) = dy * factor * x.pow(factor - static_cast(1)); } }; -template -class STanhKernel : public framework::OpKernel { - public: - void Compute(const framework::ExecutionContext& context) const override { - auto* X = context.Input("X"); - auto* Y = context.Output("Y"); - auto scale_a = static_cast(context.Attr("scale_a")); - auto scale_b = static_cast(context.Attr("scale_b")); - Y->mutable_data(context.GetPlace()); +template +struct STanhFunctor : public BaseActivationFunctor { + float scale_a; + float scale_b; + typename BaseActivationFunctor::AttrPair GetAttrs() { + return {{"scale_a", &scale_a}, {"scale_b", &scale_b}}; + } - auto x = framework::EigenVector::Flatten(*X); - auto y = framework::EigenVector::Flatten(*Y); - auto place = context.GetEigenDevice(); - y.device(place) = scale_b * (scale_a * x).tanh(); + template + void operator()(Device d, X x, Y y) const { + y.device(d) = scale_b * (scale_a * x).tanh(); } }; -template -class STanhGradKernel : public framework::OpKernel { - public: - void Compute(const framework::ExecutionContext& context) const override { - auto* X = context.Input("X"); - auto* dY = context.Input(framework::GradVarName("Y")); - auto* dX = context.Output(framework::GradVarName("X")); - auto scale_a = static_cast(context.Attr("scale_a")); - auto scale_b = static_cast(context.Attr("scale_b")); - dX->mutable_data(context.GetPlace()); - - auto dy = framework::EigenVector::Flatten(*dY); - auto x = framework::EigenVector::Flatten(*X); - auto dx = framework::EigenVector::Flatten(*dX); - auto place = context.GetEigenDevice(); +template +struct STanhGradFunctor : public BaseActivationFunctor { + float scale_a; + float scale_b; + typename BaseActivationFunctor::AttrPair GetAttrs() { + return {{"scale_a", &scale_a}, {"scale_b", &scale_b}}; + } + template + void operator()(Device d, X x, Y y, dY dy, dX dx) const { auto temp = (scale_a * x).tanh() * (scale_a * x).tanh(); - dx.device(place) = dy * scale_a * scale_b * (static_cast(1) - temp); + dx.device(d) = dy * scale_a * scale_b * (static_cast(1) - temp); } }; } // namespace operators } // namespace paddle + +#define FOR_EACH_KERNEL_FUNCTOR(__macro) \ + __macro(sigmoid, SigmoidFunctor, SigmoidGradFunctor); \ + __macro(exp, ExpFunctor, ExpGradFunctor); \ + __macro(relu, ReluFunctor, ReluGradFunctor); \ + __macro(tanh, TanhFunctor, TanhGradFunctor); \ + __macro(sqrt, SqrtFunctor, SqrtGradFunctor); \ + __macro(abs, AbsFunctor, AbsGradFunctor); \ + __macro(reciprocal, ReciprocalFunctor, ReciprocalGradFunctor); \ + __macro(log, LogFunctor, LogGradFunctor); \ + __macro(square, SquareFunctor, SquareGradFunctor); \ + __macro(brelu, BReluFunctor, BReluGradFunctor); \ + __macro(soft_relu, SoftReluFunctor, SoftReluGradFunctor); \ + __macro(pow, PowFunctor, PowGradFunctor); \ + __macro(stanh, STanhFunctor, STanhGradFunctor) From 78808b20911dd95e1a49495c99d814b59e3290c9 Mon Sep 17 00:00:00 2001 From: zchen0211 Date: Thu, 28 Sep 2017 17:27:37 -0700 Subject: [PATCH 038/170] 1 api --- paddle/operators/cond_op.cc | 4 ++-- paddle/operators/gather.cu.h | 30 ++++++++++---------------- paddle/operators/gather.h | 38 +++++++++++++-------------------- paddle/operators/gather_op.cu | 4 ++-- paddle/operators/gather_op.h | 2 +- paddle/operators/gather_test.cc | 2 +- paddle/operators/scatter.cu.h | 36 ++++++++++++------------------- paddle/operators/scatter.h | 20 ++++++----------- paddle/operators/scatter_op.cu | 4 ++-- paddle/operators/scatter_op.h | 2 +- 10 files changed, 55 insertions(+), 87 deletions(-) diff --git a/paddle/operators/cond_op.cc b/paddle/operators/cond_op.cc index 157656786a..983b5142b1 100644 --- a/paddle/operators/cond_op.cc +++ b/paddle/operators/cond_op.cc @@ -169,8 +169,8 @@ void CondOp::Run(const Scope& scope, tensor_child->Resize(dim); tensor_child->mutable_data(dim, platform::CPUPlace()); - CPUTGather(dev_ctx.GetPlace(), tensor_parent, &index_tensors[i], - tensor_child); + CPUGather(dev_ctx.GetPlace(), tensor_parent, &index_tensors[i], + tensor_child); } } diff --git a/paddle/operators/gather.cu.h b/paddle/operators/gather.cu.h index c96071e295..b400c10440 100644 --- a/paddle/operators/gather.cu.h +++ b/paddle/operators/gather.cu.h @@ -38,19 +38,6 @@ __global__ void GatherCUDAKernel(const T* params, const int* indices, T* output, } } -// Implementation of GPU copy: -template -struct GPUGather { - void operator()(const T* src, const int* index, const int slice_size, - const int index_size, T* output) { - int block = 512; - int n = slice_size * index_size; - int grid = (n + block - 1) / block; - GatherCUDAKernel<<>>(src, index, output, index_size, - slice_size); - } -}; - /** * A thin wrapper on gpu tensor * Return a new tensor from source tensor, gathered according to index @@ -59,8 +46,8 @@ struct GPUGather { * return: output tensor */ template -void GPUTGather(const Place& place, const Tensor* src, const Tensor* index, - Tensor* output) { +void GPUGather(const Place& place, const Tensor* src, const Tensor* index, + Tensor* output) { PADDLE_ENFORCE(platform::is_gpu_place(place)); // check index of shape 1-D PADDLE_ENFORCE(index->dims().size() == 1); @@ -74,10 +61,15 @@ void GPUTGather(const Place& place, const Tensor* src, const Tensor* index, int slice_size = 1; for (int i = 1; i < src_dims.size(); ++i) slice_size *= src_dims[i]; - // Gathering - GPUGather gather_functor; - gather_functor(src->data(), index->data(), slice_size, index_size, - output->data()); + const T* p_src = src->data(); + const int* p_index = index->data(); + T* p_output = output->data(); + + int block = 512; + int n = slice_size * index_size; + int grid = (n + block - 1) / block; + GatherCUDAKernel<<>>(p_src, p_index, p_output, index_size, + slice_size); } } // namespace operators diff --git a/paddle/operators/gather.h b/paddle/operators/gather.h index a3db17bd3d..cb635f6825 100644 --- a/paddle/operators/gather.h +++ b/paddle/operators/gather.h @@ -24,32 +24,18 @@ limitations under the License. */ namespace paddle { namespace operators { -// Implementation of CPU copy -template -struct CPUGather { - void operator()(const T* src, const int* indices, const int slice_size, - const int index_size, T* output) { - const size_t slice_bytes = slice_size * sizeof(T); - - for (int i = 0; i < index_size; ++i) { - int index_ = indices[i]; - memcpy(output + i * slice_size, src + index_ * slice_size, slice_bytes); - } - } -}; - /** - * A thin wrapper on cpu tensor + * A thin wrapper for gathering on cpu tensor * Return a new tensor from source tensor, gathered according to index * input[src]: type-T source Tensor * input[index]: type-int index Tensor (1-D) * return: output tensor */ template -void CPUTGather(const platform::Place& place, - const paddle::framework::Tensor* src, - const paddle::framework::Tensor* index, - paddle::framework::Tensor* output) { +void CPUGather(const platform::Place& place, + const paddle::framework::Tensor* src, + const paddle::framework::Tensor* index, + paddle::framework::Tensor* output) { PADDLE_ENFORCE(platform::is_cpu_place(place)); // check index of shape 1-D PADDLE_ENFORCE(index->dims().size() == 1); @@ -59,14 +45,20 @@ void CPUTGather(const platform::Place& place, framework::DDim output_dims(src_dims); output_dims[0] = index_size; + const T* p_src = src->data(); + const int* p_index = index->data(); + T* p_output = output->data(); + // slice size int slice_size = 1; for (int i = 1; i < src_dims.size(); ++i) slice_size *= src_dims[i]; - // Gathering - CPUGather gather_functor; - gather_functor(src->data(), index->data(), slice_size, index_size, - output->data()); + const size_t slice_bytes = slice_size * sizeof(T); + + for (int i = 0; i < index_size; ++i) { + int index_ = p_index[i]; + memcpy(p_output + i * slice_size, p_src + index_ * slice_size, slice_bytes); + } } } // namespace operators diff --git a/paddle/operators/gather_op.cu b/paddle/operators/gather_op.cu index f7533cdd64..06004614b2 100644 --- a/paddle/operators/gather_op.cu +++ b/paddle/operators/gather_op.cu @@ -32,7 +32,7 @@ class GatherOpCUDAKernel : public framework::OpKernel { output->mutable_data(ctx.GetPlace()); - GPUTGather(ctx.GetPlace(), x, index, output); + GPUGather(ctx.GetPlace(), x, index, output); } }; @@ -53,7 +53,7 @@ class GatherGradOpCUDAKernel : public framework::OpKernel { auto place = ctx.GetEigenDevice(); dxt.device(place) = dxt.constant(static_cast(0)); - GPUTScatter(ctx.GetPlace(), dO, Index, dX); + GPUScatterAssign(ctx.GetPlace(), dO, Index, dX); } }; diff --git a/paddle/operators/gather_op.h b/paddle/operators/gather_op.h index b80a4ab370..fb065b8da7 100644 --- a/paddle/operators/gather_op.h +++ b/paddle/operators/gather_op.h @@ -36,7 +36,7 @@ class GatherOpKernel : public framework::OpKernel { output->mutable_data(ctx.GetPlace()); - CPUTGather(ctx.GetPlace(), x, index, output); + CPUGather(ctx.GetPlace(), x, index, output); } }; diff --git a/paddle/operators/gather_test.cc b/paddle/operators/gather_test.cc index ea06ae2847..3c1d06ccd1 100644 --- a/paddle/operators/gather_test.cc +++ b/paddle/operators/gather_test.cc @@ -41,7 +41,7 @@ TEST(Gather, GatherData) { int* p_output = output->mutable_data(make_ddim({2, 4}), CPUPlace()); - CPUTGather(CPUPlace(), src, index, output); + CPUGather(CPUPlace(), src, index, output); for (int i = 0; i < 4; ++i) EXPECT_EQ(p_output[i], i + 4); for (int i = 4; i < 8; ++i) EXPECT_EQ(p_output[i], i - 4); diff --git a/paddle/operators/scatter.cu.h b/paddle/operators/scatter.cu.h index 82e5040305..add4791a79 100644 --- a/paddle/operators/scatter.cu.h +++ b/paddle/operators/scatter.cu.h @@ -36,20 +36,6 @@ __global__ void ScatterCUDAKernel(const T* params, const int* indices, } } -// Implementation of GPU copy: -template -struct GPUScatterAssign { - void operator()(const T* src, const int* index, const int slice_size, - const int index_size, T* output) { - int block = 512; - int n = slice_size * index_size; - int grid = (n + block - 1) / block; - // printf("grid, block: %d %d\n", grid, block); - ScatterCUDAKernel<<>>(src, index, output, index_size, - slice_size); - } -}; - /** * A thin wrapper on gpu tensor * Return a new updated tensor from source tensor, scatter-assigned according to @@ -59,10 +45,10 @@ struct GPUScatterAssign { * return: output tensor */ template -void GPUTScatter(const platform::Place& place, - const paddle::framework::Tensor* src, - const paddle::framework::Tensor* index, - paddle::framework::Tensor* output) { +void GPUScatterAssign(const platform::Place& place, + const paddle::framework::Tensor* src, + const paddle::framework::Tensor* index, + paddle::framework::Tensor* output) { PADDLE_ENFORCE(platform::is_gpu_place(place)); // check index of shape 1-D PADDLE_ENFORCE(index->dims().size() == 1); @@ -76,10 +62,16 @@ void GPUTScatter(const platform::Place& place, int slice_size = 1; for (int i = 1; i < src_dims.size(); ++i) slice_size *= src_dims[i]; - // Scatter Assign - GPUScatterAssign scatter_functor; - scatter_functor(src->data(), index->data(), slice_size, index_size, - output->data()); + const T* p_src = src->data(); + const int* p_index = index->data(); + T* p_output = output->data(); + + int block = 512; + int n = slice_size * index_size; + int grid = (n + block - 1) / block; + + ScatterCUDAKernel<<>>(p_src, p_index, p_output, index_size, + slice_size); } } // namespace operators diff --git a/paddle/operators/scatter.h b/paddle/operators/scatter.h index 670204b4dd..f895f22e28 100644 --- a/paddle/operators/scatter.h +++ b/paddle/operators/scatter.h @@ -25,19 +25,6 @@ namespace operators { using Tensor = framework::Tensor; -// Implementation of CPU copy -template -void CPUScatterAssign(const T* src, const int* index, const int slice_size, - const int index_size, T* output) { - // paddle::framework::DDim output_dims = output->dims(); - const size_t slice_bytes = slice_size * sizeof(T); - - for (int i = 0; i < index_size; ++i) { - int index_ = index[i]; - memcpy(output + index_ * slice_size, src + i * slice_size, slice_bytes); - } -} - /** * Return a updated tensor from source tensor, scattered according to index: * dst[i] = src[index[i]] @@ -70,7 +57,12 @@ void ScatterAssign(const platform::Place& place, size_t slice_size = 1; for (int i = 1; i < src_dims.size(); ++i) slice_size *= src_dims[i]; - CPUScatterAssign(p_src, p_index, slice_size, index_size, p_output); + const size_t slice_bytes = slice_size * sizeof(T); + + for (int i = 0; i < index_size; ++i) { + int index_ = p_index[i]; + memcpy(p_output + index_ * slice_size, p_src + i * slice_size, slice_bytes); + } } } // namespace operators diff --git a/paddle/operators/scatter_op.cu b/paddle/operators/scatter_op.cu index 89d23945e0..831eabdae4 100644 --- a/paddle/operators/scatter_op.cu +++ b/paddle/operators/scatter_op.cu @@ -32,7 +32,7 @@ class ScatterOpCUDAKernel : public framework::OpKernel { Out->ShareDataWith(*Ref); - GPUTScatter(ctx.GetPlace(), Updates, Index, Out); + GPUScatterAssign(ctx.GetPlace(), Updates, Index, Out); } }; @@ -51,7 +51,7 @@ class ScatterGradOpCUDAKernel : public framework::OpKernel { dRef->ShareDataWith(*dOut); dUpdates->mutable_data(ctx.GetPlace()); // Gradient by Gather: dUpdates = dO[Index] - GPUTGather(ctx.GetPlace(), dOut, Index, dUpdates); + GPUGather(ctx.GetPlace(), dOut, Index, dUpdates); } }; diff --git a/paddle/operators/scatter_op.h b/paddle/operators/scatter_op.h index 74b2718f43..771a1f2ddb 100644 --- a/paddle/operators/scatter_op.h +++ b/paddle/operators/scatter_op.h @@ -56,7 +56,7 @@ class ScatterGradientOpKernel : public framework::OpKernel { dRef->ShareDataWith(*dOut); dUpdates->mutable_data(ctx.GetPlace()); // Gradient by Gather: dUpdates += dO[Index] - CPUTGather(ctx.GetPlace(), dOut, Index, dUpdates); + CPUGather(ctx.GetPlace(), dOut, Index, dUpdates); } }; From 61cc3ae4d13a798f341ceb5b2240b92526b3f43f Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Thu, 28 Sep 2017 11:52:03 -0700 Subject: [PATCH 039/170] Stablize elementwise_mul by using double precision --- paddle/pybind/pybind.cc | 16 +++-- paddle/pybind/tensor_py.h | 15 ++++- python/paddle/v2/framework/tests/op_test.py | 60 +++++++++++++------ .../tests/test_elementwise_mul_op.py | 32 +++++----- 4 files changed, 78 insertions(+), 45 deletions(-) diff --git a/paddle/pybind/pybind.cc b/paddle/pybind/pybind.cc index d85bf6c7fa..f4121e9d71 100644 --- a/paddle/pybind/pybind.cc +++ b/paddle/pybind/pybind.cc @@ -77,20 +77,18 @@ PYBIND11_PLUGIN(core) { }) .def("set", PyCPUTensorSetFromArray) .def("set", PyCPUTensorSetFromArray) + .def("set", PyCPUTensorSetFromArray) #ifndef PADDLE_ONLY_CPU .def("set", PyCUDATensorSetFromArray) .def("set", PyCUDATensorSetFromArray) + .def("set", PyCUDATensorSetFromArray) #endif .def("shape", [](Tensor &self) { return vectorize(self.dims()); }) - .def("set_float_element", - [](Tensor &self, size_t offset, float f) { - // TODO(yuyang18): Only support GPU now. - self.data()[offset] = f; - }) - .def("get_float_element", [](Tensor &self, size_t offset) -> float { - // TODO(yuyang18): Only support GPU now. - return self.data()[offset]; - }); + .def("set_float_element", TensorSetElement) + .def("get_float_element", TensorGetElement) + .def("set_double_element", TensorSetElement) + .def("get_double_element", TensorGetElement) + .def("dtype", [](Tensor &self) { return ToDataType(self.type()); }); py::class_(m, "LoDTensor") .def_buffer( diff --git a/paddle/pybind/tensor_py.h b/paddle/pybind/tensor_py.h index 10621e90ee..3e3e6bc031 100644 --- a/paddle/pybind/tensor_py.h +++ b/paddle/pybind/tensor_py.h @@ -73,10 +73,23 @@ struct CastToPyBufferImpl { }; } // namespace details inline py::buffer_info CastToPyBuffer(framework::Tensor &tensor) { - auto buffer_info = details::CastToPyBufferImpl()(tensor); + auto buffer_info = + details::CastToPyBufferImpl()(tensor); return buffer_info; } +template +T TensorGetElement(framework::Tensor &self, size_t offset) { + PADDLE_ENFORCE(platform::is_cpu_place(self.place())); + return self.data()[offset]; +} + +template +void TensorSetElement(framework::Tensor &self, size_t offset, T elem) { + PADDLE_ENFORCE(platform::is_cpu_place(self.place())); + self.data()[offset] = elem; +} + template void PyCPUTensorSetFromArray( framework::Tensor &self, diff --git a/python/paddle/v2/framework/tests/op_test.py b/python/paddle/v2/framework/tests/op_test.py index 89979044f2..70ae50d401 100644 --- a/python/paddle/v2/framework/tests/op_test.py +++ b/python/paddle/v2/framework/tests/op_test.py @@ -69,24 +69,27 @@ def set_input(scope, op, inputs, place): def set_output_grad(scope, op, outputs, place): + def __set_tensor__(name): + out_tensor = scope.find_var(name).get_tensor() + grad_tensor = scope.new_var(grad_var_name(name)).get_tensor() + out_dtype = out_tensor.dtype() + if out_dtype == core.DataType.FP64: + data = np.ones(out_tensor.shape(), dtype=np.float64) + elif out_dtype == core.DataType.FP32: + data = np.ones(out_tensor.shape(), dtype=np.float32) + else: + raise ValueError("Not supported data type " + str(out_dtype)) + + grad_tensor.set(data, place) + for out_name, out_dup in Operator.get_op_outputs(op.type()): if out_name in outputs: if out_dup: sub_out = outputs[out_name] for sub_out_name, _ in sub_out: - out_tensor = scope.find_var(sub_out_name).get_tensor() - grad_tensor = scope.new_var(grad_var_name( - sub_out_name)).get_tensor() - grad_tensor.set_dims(out_tensor.shape()) - data = np.ones(out_tensor.shape(), dtype=np.float32) - grad_tensor.set(data, place) + __set_tensor__(sub_out_name) else: - out_tensor = scope.find_var(out_name).get_tensor() - grad_tensor = scope.new_var(grad_var_name(out_name)).get_tensor( - ) - grad_tensor.set_dims(out_tensor.shape()) - data = np.ones(out_tensor.shape(), dtype=np.float32) - grad_tensor.set(data, place) + __set_tensor__(out_name) def get_numeric_gradient(scope, @@ -96,7 +99,6 @@ def get_numeric_gradient(scope, output_names, delta=0.005, in_place=False): - set_input(scope, op, inputs, core.CPUPlace()) tensor_to_check = scope.find_var(input_to_check).get_tensor() @@ -115,7 +117,29 @@ def get_numeric_gradient(scope, tensor_to_check = scope.find_var(input_to_check).get_tensor() tensor_size = product(tensor_to_check.get_dims()) - gradient_flat = np.zeros(shape=(tensor_size, ), dtype='float32') + tensor_to_check_dtype = tensor_to_check.dtype() + if tensor_to_check_dtype == core.DataType.FP32: + tensor_to_check_dtype = np.float32 + elif tensor_to_check_dtype == core.DataType.FP64: + tensor_to_check_dtype = np.float64 + else: + raise ValueError("Not supported data type " + str( + tensor_to_check_dtype)) + + gradient_flat = np.zeros(shape=(tensor_size, ), dtype=tensor_to_check_dtype) + + def __get_elem__(tensor, i): + if tensor_to_check_dtype == np.float32: + return tensor.get_float_element(i) + else: + return tensor.get_double_element(i) + + def __set_elem__(tensor, i, e): + if tensor_to_check_dtype == np.float32: + tensor.set_float_element(i, e) + else: + tensor.set_double_element(i, e) + # we only compute gradient of one element each time. # we use a for loop to compute the gradient of every element. for i in xrange(tensor_size): @@ -123,20 +147,20 @@ def get_numeric_gradient(scope, set_input(scope, op, inputs, core.CPUPlace()) # get one input element throw it's index i. - origin = tensor_to_check.get_float_element(i) + origin = __get_elem__(tensor_to_check, i) # add delta to it, run op and then get the sum of the result tensor. x_pos = origin + delta - tensor_to_check.set_float_element(i, x_pos) + __set_elem__(tensor_to_check, i, x_pos) y_pos = get_output() if in_place: set_input(scope, op, inputs, core.CPUPlace()) x_neg = origin - delta - tensor_to_check.set_float_element(i, x_neg) + __set_elem__(tensor_to_check, i, x_neg) y_neg = get_output() - tensor_to_check.set_float_element(i, origin) + __set_elem__(tensor_to_check, i, origin) gradient_flat[i] = (y_pos - y_neg) / delta / 2 return gradient_flat.reshape(tensor_to_check.get_dims()) diff --git a/python/paddle/v2/framework/tests/test_elementwise_mul_op.py b/python/paddle/v2/framework/tests/test_elementwise_mul_op.py index cee4385a81..261ca9cb3d 100644 --- a/python/paddle/v2/framework/tests/test_elementwise_mul_op.py +++ b/python/paddle/v2/framework/tests/test_elementwise_mul_op.py @@ -7,8 +7,8 @@ class ElementwiseMulOp(OpTest): def setUp(self): self.op_type = "elementwise_mul" self.inputs = { - 'X': np.random.uniform(0.1, 1, [13, 17]).astype("float32"), - 'Y': np.random.uniform(0.1, 1, [13, 17]).astype("float32") + 'X': np.random.uniform(0.1, 1, [13, 17]).astype("float64"), + 'Y': np.random.uniform(0.1, 1, [13, 17]).astype("float64") } self.outputs = {'Out': np.multiply(self.inputs['X'], self.inputs['Y'])} @@ -16,23 +16,21 @@ class ElementwiseMulOp(OpTest): self.check_output() def test_check_grad_normal(self): - self.check_grad(['X', 'Y'], 'Out', max_relative_error=0.1) + self.check_grad(['X', 'Y'], 'Out') def test_check_grad_ingore_x(self): - self.check_grad( - ['Y'], 'Out', max_relative_error=0.1, no_grad_set=set("X")) + self.check_grad(['Y'], 'Out', no_grad_set=set("X")) def test_check_grad_ingore_y(self): - self.check_grad( - ['X'], 'Out', max_relative_error=0.1, no_grad_set=set('Y')) + self.check_grad(['X'], 'Out', no_grad_set=set('Y')) class TestElementwiseMulOp_Vector(ElementwiseMulOp): def setUp(self): self.op_type = "elementwise_mul" self.inputs = { - 'X': np.random.random((32, )).astype("float32"), - 'Y': np.random.random((32, )).astype("float32") + 'X': np.random.random((32, )).astype("float64"), + 'Y': np.random.random((32, )).astype("float64") } self.outputs = {'Out': np.multiply(self.inputs['X'], self.inputs['Y'])} @@ -41,8 +39,8 @@ class TestElementwiseMulOp_broadcast_0(ElementwiseMulOp): def setUp(self): self.op_type = "elementwise_mul" self.inputs = { - 'X': np.random.rand(2, 3, 4).astype(np.float32), - 'Y': np.random.rand(2).astype(np.float32) + 'X': np.random.rand(2, 3, 4).astype(np.float64), + 'Y': np.random.rand(2).astype(np.float64) } self.attrs = {'axis': 0} @@ -55,8 +53,8 @@ class TestElementwiseMulOp_broadcast_1(ElementwiseMulOp): def setUp(self): self.op_type = "elementwise_mul" self.inputs = { - 'X': np.random.rand(2, 3, 4).astype(np.float32), - 'Y': np.random.rand(3).astype(np.float32) + 'X': np.random.rand(2, 3, 4).astype(np.float64), + 'Y': np.random.rand(3).astype(np.float64) } self.attrs = {'axis': 1} @@ -69,8 +67,8 @@ class TestElementwiseMulOp_broadcast_2(ElementwiseMulOp): def setUp(self): self.op_type = "elementwise_mul" self.inputs = { - 'X': np.random.rand(2, 3, 4).astype(np.float32), - 'Y': np.random.rand(4).astype(np.float32) + 'X': np.random.rand(2, 3, 4).astype(np.float64), + 'Y': np.random.rand(4).astype(np.float64) } self.outputs = { @@ -82,8 +80,8 @@ class TestElementwiseMulOp_broadcast_3(ElementwiseMulOp): def setUp(self): self.op_type = "elementwise_mul" self.inputs = { - 'X': np.random.rand(2, 3, 4, 5).astype(np.float32), - 'Y': np.random.rand(3, 4).astype(np.float32) + 'X': np.random.rand(2, 3, 4, 5).astype(np.float64), + 'Y': np.random.rand(3, 4).astype(np.float64) } self.attrs = {'axis': 1} From 54892c079735aaffafc7388486482e06ff139439 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Thu, 28 Sep 2017 11:59:17 -0700 Subject: [PATCH 040/170] Simplify op_test --- python/paddle/v2/framework/tests/op_test.py | 42 +++++++++------------ 1 file changed, 18 insertions(+), 24 deletions(-) diff --git a/python/paddle/v2/framework/tests/op_test.py b/python/paddle/v2/framework/tests/op_test.py index 70ae50d401..23794151bd 100644 --- a/python/paddle/v2/framework/tests/op_test.py +++ b/python/paddle/v2/framework/tests/op_test.py @@ -12,17 +12,19 @@ def grad_var_name(var_name): def create_op(scope, op_type, inputs, outputs, attrs): kwargs = dict() + def __create_var__(name, var_name): + scope.new_var(var_name) + kwargs[name].append(var_name) + for in_name, in_dup in Operator.get_op_inputs(op_type): if in_name in inputs: kwargs[in_name] = [] if in_dup: sub_in = inputs[in_name] for sub_in_name, _ in sub_in: - var = scope.new_var(sub_in_name) - kwargs[in_name].append(sub_in_name) + __create_var__(in_name, sub_in_name) else: - var = scope.new_var(in_name) - kwargs[in_name].append(in_name) + __create_var__(in_name, in_name) for out_name, out_dup in Operator.get_op_outputs(op_type): if out_name in outputs: @@ -30,11 +32,9 @@ def create_op(scope, op_type, inputs, outputs, attrs): if out_dup: sub_out = outputs[out_name] for sub_out_name, _ in sub_out: - var = scope.new_var(sub_out_name) - kwargs[out_name].append(sub_out_name) + __create_var__(out_name, sub_out_name) else: - var = scope.new_var(out_name) - kwargs[out_name].append(out_name) + __create_var__(out_name, out_name) for attr_name in Operator.get_op_attr_names(op_type): if attr_name in attrs: @@ -44,28 +44,22 @@ def create_op(scope, op_type, inputs, outputs, attrs): def set_input(scope, op, inputs, place): + def __set_input__(var_name, var): + tensor = scope.find_var(var_name).get_tensor() + if isinstance(var, tuple): + tensor.set_lod(var[1]) + var = var[0] + tensor.set_dims(var.shape) + tensor.set(var, place) + for in_name, in_dup in Operator.get_op_inputs(op.type()): if in_name in inputs: if in_dup: sub_in = inputs[in_name] for sub_in_name, sub_in_val in sub_in: - var = scope.find_var(sub_in_name) - tensor = var.get_tensor() - sub_in_array = sub_in_val[0] \ - if isinstance(sub_in_val, tuple) else sub_in_val - tensor.set_dims(sub_in_array.shape) - tensor.set(sub_in_array, place) - if isinstance(sub_in_val, tuple): - tensor.set_lod(sub_in_val[1]) + __set_input__(sub_in_name, sub_in_val) else: - var = scope.find_var(in_name) - tensor = var.get_tensor() - in_val = inputs[in_name] - in_array = in_val[0] if isinstance(in_val, tuple) else in_val - tensor.set_dims(in_array.shape) - tensor.set(in_array, place) - if isinstance(in_val, tuple): - tensor.set_lod(in_val[1]) + __set_input__(in_name, inputs[in_name]) def set_output_grad(scope, op, outputs, place): From 279178e457dfb12c15ddb4e51d8f75e75ad6db1f Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Thu, 28 Sep 2017 16:45:13 -0700 Subject: [PATCH 041/170] Fix bug in test_prelu and test_xe They were using float64 for FP32 kernel before. --- python/paddle/v2/framework/tests/test_cross_entropy_op.py | 2 +- python/paddle/v2/framework/tests/test_prelu_op.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/paddle/v2/framework/tests/test_cross_entropy_op.py b/python/paddle/v2/framework/tests/test_cross_entropy_op.py index 1de514dff4..4ea14da7fd 100644 --- a/python/paddle/v2/framework/tests/test_cross_entropy_op.py +++ b/python/paddle/v2/framework/tests/test_cross_entropy_op.py @@ -80,7 +80,7 @@ class TestCrossEntropyOp3(OpTest): cross_entropy2 = (-label * np.log(X)).sum( axis=1, keepdims=True).astype("float32") - self.inputs = {"X": X, "Label": label} + self.inputs = {"X": X, "Label": label.astype(np.float32)} self.outputs = {"Y": cross_entropy} self.attrs = {"softLabel": True} diff --git a/python/paddle/v2/framework/tests/test_prelu_op.py b/python/paddle/v2/framework/tests/test_prelu_op.py index 676fd9f7c5..7be932ac8f 100644 --- a/python/paddle/v2/framework/tests/test_prelu_op.py +++ b/python/paddle/v2/framework/tests/test_prelu_op.py @@ -17,7 +17,7 @@ class PReluTest(OpTest): x_np_sign = np.sign(x_np) x_np = x_np_sign * np.maximum(x_np, .005) - alpha_np = np.array([.1]) + alpha_np = np.array([.1], dtype="float32") self.inputs = {'X': x_np, 'Alpha': alpha_np} out_np = np.maximum(self.inputs['X'], 0.) out_np = out_np + np.minimum(self.inputs['X'], From f60f0eae11f9712f2f7955bb351f82dc2c2c412c Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Thu, 28 Sep 2017 20:00:23 -0700 Subject: [PATCH 042/170] Using double precision to stablize lstm gradient check --- paddle/operators/lstm_unit_op.cc | 13 +++++++------ paddle/operators/lstm_unit_op.cu | 14 ++++++++------ paddle/operators/lstm_unit_op.h | 8 ++++---- .../paddle/v2/framework/tests/test_lstm_unit_op.py | 6 +++--- 4 files changed, 22 insertions(+), 19 deletions(-) diff --git a/paddle/operators/lstm_unit_op.cc b/paddle/operators/lstm_unit_op.cc index bd75b001cb..dad56731de 100644 --- a/paddle/operators/lstm_unit_op.cc +++ b/paddle/operators/lstm_unit_op.cc @@ -47,7 +47,6 @@ class LstmUnitOp : public framework::OperatorWithKernel { } }; -template class LstmUnitOpMaker : public framework::OpProtoAndCheckerMaker { public: LstmUnitOpMaker(framework::OpProto* proto, @@ -68,7 +67,7 @@ Equation: H = C * sigm(o) )DOC"); - AddAttr("forget_bias", "The forget bias of Lstm Unit.") + AddAttr("forget_bias", "The forget bias of Lstm Unit.") .SetDefault(0.0); } }; @@ -93,9 +92,11 @@ class LstmUnitGradOp : public framework::OperatorWithKernel { } // namespace paddle namespace ops = paddle::operators; -REGISTER_OP(lstm_unit, ops::LstmUnitOp, ops::LstmUnitOpMaker, - lstm_unit_grad, ops::LstmUnitGradOp); +REGISTER_OP(lstm_unit, ops::LstmUnitOp, ops::LstmUnitOpMaker, lstm_unit_grad, + ops::LstmUnitGradOp); REGISTER_OP_CPU_KERNEL(lstm_unit, - ops::LstmUnitKernel); + ops::LstmUnitKernel, + ops::LstmUnitKernel); REGISTER_OP_CPU_KERNEL( - lstm_unit_grad, ops::LstmUnitGradKernel); + lstm_unit_grad, ops::LstmUnitGradKernel, + ops::LstmUnitGradKernel); diff --git a/paddle/operators/lstm_unit_op.cu b/paddle/operators/lstm_unit_op.cu index b1db0d5322..49ea550b6f 100644 --- a/paddle/operators/lstm_unit_op.cu +++ b/paddle/operators/lstm_unit_op.cu @@ -89,7 +89,7 @@ __global__ void LSTMUnitGradientKernel(const int nthreads, const int dim, } } -template +template class LstmUnitOpCUDAKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { @@ -101,7 +101,7 @@ class LstmUnitOpCUDAKernel : public framework::OpKernel { auto* c_tensor = ctx.Output("C"); auto* h_tensor = ctx.Output("H"); - auto forget_bias = static_cast(ctx.Attr("forget_bias")); + auto forget_bias = static_cast(ctx.Attr("forget_bias")); int b_size = c_tensor->dims()[0]; int D = c_tensor->dims()[1]; @@ -120,7 +120,7 @@ class LstmUnitOpCUDAKernel : public framework::OpKernel { } }; -template +template class LstmUnitGradOpCUDAKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { @@ -153,7 +153,7 @@ class LstmUnitGradOpCUDAKernel : public framework::OpKernel { int N = c_tensor->dims()[0]; int D = c_tensor->dims()[1]; - auto forget_bias = static_cast(ctx.Attr("forget_bias")); + auto forget_bias = static_cast(ctx.Attr("forget_bias")); int block = 512; int n = N * D; @@ -169,5 +169,7 @@ class LstmUnitGradOpCUDAKernel : public framework::OpKernel { } // namespace paddle namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL(lstm_unit, ops::LstmUnitOpCUDAKernel); -REGISTER_OP_GPU_KERNEL(lstm_unit_grad, ops::LstmUnitGradOpCUDAKernel); +REGISTER_OP_GPU_KERNEL(lstm_unit, ops::LstmUnitOpCUDAKernel, + ops::LstmUnitOpCUDAKernel); +REGISTER_OP_GPU_KERNEL(lstm_unit_grad, ops::LstmUnitGradOpCUDAKernel, + ops::LstmUnitGradOpCUDAKernel); diff --git a/paddle/operators/lstm_unit_op.h b/paddle/operators/lstm_unit_op.h index 0dc9a7d9a7..a0ff498c1d 100644 --- a/paddle/operators/lstm_unit_op.h +++ b/paddle/operators/lstm_unit_op.h @@ -32,7 +32,7 @@ inline T tanh(T x) { return 2. * sigmoid(2. * x) - 1.; } -template +template class LstmUnitKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { @@ -44,7 +44,7 @@ class LstmUnitKernel : public framework::OpKernel { auto* c_tensor = ctx.Output("C"); auto* h_tensor = ctx.Output("H"); - auto forget_bias = static_cast(ctx.Attr("forget_bias")); + auto forget_bias = static_cast(ctx.Attr("forget_bias")); int b_size = c_tensor->dims()[0]; int D = c_tensor->dims()[1]; @@ -75,7 +75,7 @@ class LstmUnitKernel : public framework::OpKernel { } }; -template +template class LstmUnitGradKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { @@ -108,7 +108,7 @@ class LstmUnitGradKernel : public framework::OpKernel { int N = c_tensor->dims()[0]; int D = c_tensor->dims()[1]; - auto forget_bias = static_cast(ctx.Attr("forget_bias")); + auto forget_bias = static_cast(ctx.Attr("forget_bias")); for (int n = 0; n < N; ++n) { for (int d = 0; d < D; ++d) { diff --git a/python/paddle/v2/framework/tests/test_lstm_unit_op.py b/python/paddle/v2/framework/tests/test_lstm_unit_op.py index 8ce65bfc31..365ee560e1 100644 --- a/python/paddle/v2/framework/tests/test_lstm_unit_op.py +++ b/python/paddle/v2/framework/tests/test_lstm_unit_op.py @@ -14,8 +14,8 @@ def tanh_np(x): class LstmUnitTest(OpTest): def setUp(self): self.op_type = "lstm_unit" - x_np = np.random.normal(size=(5, 16)).astype("float32") - c_np = np.random.normal(size=(5, 4)).astype("float32") + x_np = np.random.normal(size=(5, 16)).astype("float64") + c_np = np.random.normal(size=(5, 4)).astype("float64") i_np, f_np, o_np, j_np = np.split(x_np, 4, axis=1) forget_bias_np = 0. self.attrs = {'forget_bias': 0.} @@ -31,7 +31,7 @@ class LstmUnitTest(OpTest): self.check_output() def test_check_grad(self): - self.check_grad(['X', 'C_prev'], ['C', 'H'], max_relative_error=0.01) + self.check_grad(['X', 'C_prev'], ['C', 'H']) if __name__ == "__main__": From e1e3859e8817adfa733a7f963963c5fe1d792b16 Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Fri, 29 Sep 2017 10:15:56 +0800 Subject: [PATCH 043/170] remove custom attr checker and fix code format --- paddle/operators/math/pooling.cc | 66 ++++++++-------- paddle/operators/math/pooling.cu | 132 ++++++++++++++++--------------- paddle/operators/math/pooling.h | 11 ++- paddle/operators/pool_op.cc | 124 ++++++++--------------------- paddle/operators/pool_op.h | 30 ++++--- 5 files changed, 153 insertions(+), 210 deletions(-) diff --git a/paddle/operators/math/pooling.cc b/paddle/operators/math/pooling.cc index 1918c8b169..3b706529d8 100644 --- a/paddle/operators/math/pooling.cc +++ b/paddle/operators/math/pooling.cc @@ -24,7 +24,7 @@ class Pool2dFunctor { void operator()(const platform::DeviceContext& context, const framework::Tensor& input, framework::Tensor& output, std::vector& ksize, std::vector& strides, - std::vector& paddings, PoolProcess pool_compute) { + std::vector& paddings, PoolProcess pool_process) { const int batch_size = input.dims()[0]; const int input_height = input.dims()[2]; const int input_width = input.dims()[3]; @@ -54,14 +54,15 @@ class Pool2dFunctor { int wstart = pw * stride_width - padding_width; int wend = std::min(wstart + ksize_width, input_width); wstart = std::max(wstart, 0); - T ele = pool_compute.initial(); + + T ele = pool_process.initial(); for (int h = hstart; h < hend; ++h) { for (int w = wstart; w < wend; ++w) { - pool_compute.compute(ele, input_data[h * input_width + w]); + pool_process.compute(ele, input_data[h * input_width + w]); } } int pool_size = (hend - hstart) * (wend - wstart); - pool_compute.finalize(ele, (static_cast(pool_size))); + pool_process.finalize(ele, (static_cast(pool_size))); output_data[ph * output_width + pw] = ele; } } @@ -80,7 +81,7 @@ class Pool2dGradFunctor { const framework::Tensor& output, const framework::Tensor& output_grad, std::vector& ksize, std::vector& strides, std::vector& paddings, - PoolProcess pool_compute) { + PoolProcess pool_grad_process) { const int batch_size = input.dims()[0]; const int input_height = input.dims()[2]; const int input_width = input.dims()[3]; @@ -115,11 +116,12 @@ class Pool2dGradFunctor { float scale = 1.0 / pool_size; for (int h = hstart; h < hend; ++h) { for (int w = wstart; w < wend; ++w) { - pool_compute.compute(input_data[h * input_width + w], - output_data[ph * output_width + pw], - output_grad_data[ph * output_width + pw], - input_grad_data[h * input_width + w], - static_cast(scale)); + pool_grad_process.compute( + input_data[h * input_width + w], + output_data[ph * output_width + pw], + output_grad_data[ph * output_width + pw], + input_grad_data[h * input_width + w], + static_cast(scale)); } } } @@ -198,21 +200,21 @@ template class MaxPool2dGradFunctor; // template class MaxPool2dGradFunctor; template class Pool2dFunctor, float>; + paddle::operators::math::MaxPool, float>; template class Pool2dFunctor, float>; + paddle::operators::math::AvgPool, float>; template class Pool2dGradFunctor< - platform::CPUPlace, paddle::operators::math::maxPoolGrad, float>; + platform::CPUPlace, paddle::operators::math::MaxPoolGrad, float>; template class Pool2dGradFunctor< - platform::CPUPlace, paddle::operators::math::avgPoolGrad, float>; + platform::CPUPlace, paddle::operators::math::AvgPoolGrad, float>; template class Pool2dFunctor, double>; + paddle::operators::math::MaxPool, double>; template class Pool2dFunctor, double>; + paddle::operators::math::AvgPool, double>; template class Pool2dGradFunctor< - platform::CPUPlace, paddle::operators::math::maxPoolGrad, double>; + platform::CPUPlace, paddle::operators::math::MaxPoolGrad, double>; template class Pool2dGradFunctor< - platform::CPUPlace, paddle::operators::math::avgPoolGrad, double>; + platform::CPUPlace, paddle::operators::math::AvgPoolGrad, double>; template class Pool3dFunctor { @@ -220,7 +222,7 @@ class Pool3dFunctor { void operator()(const platform::DeviceContext& context, const framework::Tensor& input, framework::Tensor& output, std::vector& ksize, std::vector& strides, - std::vector& paddings, PoolProcess pool_compute) { + std::vector& paddings, PoolProcess pool_process) { const int batch_size = input.dims()[0]; const int input_depth = input.dims()[2]; const int input_height = input.dims()[3]; @@ -260,11 +262,11 @@ class Pool3dFunctor { int wend = std::min(wstart + ksize_width, input_width); wstart = std::max(wstart, 0); int output_idx = (pd * output_height + ph) * output_width + pw; - T ele = pool_compute.initial(); + T ele = pool_process.initial(); for (int d = dstart; d < dend; ++d) { for (int h = hstart; h < hend; ++h) { for (int w = wstart; w < wend; ++w) { - pool_compute.compute( + pool_process.compute( ele, input_data[(d * input_height + h) * input_width + w]); } @@ -272,7 +274,7 @@ class Pool3dFunctor { } int pool_size = (dend - dstart) * (hend - hstart) * (wend - wstart); - pool_compute.finalize(ele, static_cast(pool_size)); + pool_process.finalize(ele, static_cast(pool_size)); output_data[output_idx] = ele; } } @@ -292,7 +294,7 @@ class Pool3dGradFunctor { const framework::Tensor& output, const framework::Tensor& output_grad, std::vector& ksize, std::vector& strides, std::vector& paddings, - PoolProcess pool_compute) { + PoolProcess pool_grad_process) { const int batch_size = input.dims()[0]; const int input_depth = input.dims()[2]; const int input_height = input.dims()[3]; @@ -343,7 +345,7 @@ class Pool3dGradFunctor { int input_idx = (d * input_height + h) * input_width + w; int output_idx = (pd * output_height + ph) * output_width + pw; - pool_compute.compute( + pool_grad_process.compute( input_data[input_idx], output_data[output_idx], output_grad_data[output_idx], input_grad_data[input_idx], static_cast(scale)); @@ -441,21 +443,21 @@ template class MaxPool3dGradFunctor; // template class MaxPool3dGradFunctor; template class Pool3dFunctor, float>; + paddle::operators::math::MaxPool, float>; template class Pool3dFunctor, float>; + paddle::operators::math::AvgPool, float>; template class Pool3dGradFunctor< - platform::CPUPlace, paddle::operators::math::maxPoolGrad, float>; + platform::CPUPlace, paddle::operators::math::MaxPoolGrad, float>; template class Pool3dGradFunctor< - platform::CPUPlace, paddle::operators::math::avgPoolGrad, float>; + platform::CPUPlace, paddle::operators::math::AvgPoolGrad, float>; template class Pool3dFunctor, double>; + paddle::operators::math::MaxPool, double>; template class Pool3dFunctor, double>; + paddle::operators::math::AvgPool, double>; template class Pool3dGradFunctor< - platform::CPUPlace, paddle::operators::math::maxPoolGrad, double>; + platform::CPUPlace, paddle::operators::math::MaxPoolGrad, double>; template class Pool3dGradFunctor< - platform::CPUPlace, paddle::operators::math::avgPoolGrad, double>; + platform::CPUPlace, paddle::operators::math::AvgPoolGrad, double>; } // namespace math } // namespace operators } // namespace paddle diff --git a/paddle/operators/math/pooling.cu b/paddle/operators/math/pooling.cu index 4164920035..8aeccd1f8e 100644 --- a/paddle/operators/math/pooling.cu +++ b/paddle/operators/math/pooling.cu @@ -20,14 +20,16 @@ namespace operators { namespace math { template -__global__ void KernelPool2dForward( - const int nthreads, const T* input_data, T* output_data, const int channels, - const int input_height, const int input_width, const int output_height, - const int output_width, const int ksize_height, const int ksize_width, - const int stride_height, const int stride_width, const int padding_height, - const int padding_width, PoolProcess pool_compute) { - int index = blockIdx.x * blockDim.x + threadIdx.x; - if (index < nthreads) { +__global__ void KernelPool2D(const int nthreads, const T* input_data, + T* output_data, const int channels, + const int input_height, const int input_width, + const int output_height, const int output_width, + const int ksize_height, const int ksize_width, + const int stride_height, const int stride_width, + const int padding_height, const int padding_width, + PoolProcess pool_process) { + for (int index = blockIdx.x * blockDim.x + threadIdx.x; index < nthreads; + index += blockDim.x * gridDim.x) { int pw = index % output_width; int ph = (index / output_width) % output_height; int c = (index / output_width / output_height) % channels; @@ -42,28 +44,28 @@ __global__ void KernelPool2dForward( wstart = max(wstart, 0); input_data += (batch_idx * channels + c) * input_height * input_width; - T ele = pool_compute.initial(); + T ele = pool_process.initial(); for (int h = hstart; h < hend; ++h) { for (int w = wstart; w < wend; ++w) { - pool_compute.compute(ele, input_data[h * input_width + w]); + pool_process.compute(ele, input_data[h * input_width + w]); } } int pool_size = (hend - hstart) * (wend - wstart); - pool_compute.finalize(ele, (static_cast(pool_size))); + pool_process.finalize(ele, (static_cast(pool_size))); output_data[index] = ele; } } template -__global__ void KernelPool2dBackward( +__global__ void KernelPool2DGrad( const int nthreads, const T* input_data, const T* output_data, const T* output_grad, T* input_grad, const int channels, const int input_height, const int input_width, const int output_height, const int output_width, const int ksize_height, const int ksize_width, const int stride_height, const int stride_width, const int padding_height, - const int padding_width, PoolProcess pool_compute) { - int index = blockIdx.x * blockDim.x + threadIdx.x; - if (index < nthreads) { + const int padding_width, PoolProcess pool_process) { + for (int index = blockIdx.x * blockDim.x + threadIdx.x; index < nthreads; + index += blockDim.x * gridDim.x) { int offsetW = index % input_width + padding_width; int offsetH = (index / input_width) % input_height + padding_height; int offsetC = (index / input_width / input_height) % channels; @@ -93,7 +95,7 @@ __global__ void KernelPool2dBackward( wstart = max(wstart, 0); int pool_size = (hend - hstart) * (wend - wstart); int output_sub_idx = ph * output_width + pw; - pool_compute.compute(input, output_data[output_sub_idx], + pool_process.compute(input, output_data[output_sub_idx], output_grad[output_sub_idx], gradient, static_cast(1.0 / pool_size)); } @@ -103,15 +105,15 @@ __global__ void KernelPool2dBackward( } template -__global__ void KernelMaxPool2dBackward( +__global__ void KernelMaxPool2DGrad( const int nthreads, const T* input_data, const T* output_data, const T* output_grad, T* input_grad, const int channels, const int input_height, const int input_width, const int output_height, const int output_width, const int ksize_height, const int ksize_width, const int stride_height, const int stride_width, const int padding_height, const int padding_width) { - int index = blockIdx.x * blockDim.x + threadIdx.x; - if (index < nthreads) { + for (int index = blockIdx.x * blockDim.x + threadIdx.x; index < nthreads; + index += blockDim.x * gridDim.x) { int pw = index % output_width; int ph = (index / output_width) % output_height; int c = (index / output_width / output_height) % channels; @@ -153,7 +155,7 @@ class Pool2dFunctor { void operator()(const platform::DeviceContext& context, const framework::Tensor& input, framework::Tensor& output, std::vector& ksize, std::vector& strides, - std::vector& paddings, PoolProcess pool_compute) { + std::vector& paddings, PoolProcess pool_process) { const int batch_size = input.dims()[0]; const int input_channels = input.dims()[1]; const int input_height = input.dims()[2]; @@ -176,7 +178,7 @@ class Pool2dFunctor { dim3 threads(1024, 1); dim3 grid(blocks, 1); - KernelPool2dForward< + KernelPool2D< PoolProcess, T><<(context) @@ -184,7 +186,7 @@ class Pool2dFunctor { input_height, input_width, output_height, output_width, ksize_height, ksize_width, stride_height, stride_width, padding_height, - padding_width, pool_compute); + padding_width, pool_process); } }; @@ -196,7 +198,7 @@ class Pool2dGradFunctor { const framework::Tensor& output, const framework::Tensor& output_grad, std::vector& ksize, std::vector& strides, std::vector& paddings, - PoolProcess pool_compute) { + PoolProcess pool_process) { const int batch_size = input.dims()[0]; const int input_channels = input.dims()[1]; const int input_height = input.dims()[2]; @@ -220,7 +222,7 @@ class Pool2dGradFunctor { dim3 threads(1024, 1); dim3 grid(blocks, 1); - KernelPool2dBackward< + KernelPool2DGrad< PoolProcess, T><<(context) @@ -228,7 +230,7 @@ class Pool2dGradFunctor { nthreads, input_data, output_data, output_grad_data, input_grad_data, input_channels, input_height, input_width, output_height, output_width, ksize_height, ksize_width, stride_height, stride_width, padding_height, - padding_width, pool_compute); + padding_width, pool_process); } }; @@ -264,7 +266,7 @@ class MaxPool2dGradFunctor { dim3 threads(1024, 1); dim3 grid(blocks, 1); - KernelMaxPool2dBackward< + KernelMaxPool2DGrad< T><<(context) .stream()>>>( @@ -276,35 +278,37 @@ class MaxPool2dGradFunctor { }; template class MaxPool2dGradFunctor; -// template class MaxPool2dGradFunctor; +// template class MaxPool2dGradFunctor; // The +// 64-bit floating-point version of atomicAdd() is only supported by devices of +// compute capability 6.x and higher. template class Pool2dFunctor, float>; + paddle::operators::math::MaxPool, float>; template class Pool2dFunctor, float>; + paddle::operators::math::AvgPool, float>; template class Pool2dGradFunctor< - platform::GPUPlace, paddle::operators::math::maxPoolGrad, float>; + platform::GPUPlace, paddle::operators::math::MaxPoolGrad, float>; template class Pool2dGradFunctor< - platform::GPUPlace, paddle::operators::math::avgPoolGrad, float>; + platform::GPUPlace, paddle::operators::math::AvgPoolGrad, float>; template class Pool2dFunctor, double>; + paddle::operators::math::MaxPool, double>; template class Pool2dFunctor, double>; + paddle::operators::math::AvgPool, double>; template class Pool2dGradFunctor< - platform::GPUPlace, paddle::operators::math::maxPoolGrad, double>; + platform::GPUPlace, paddle::operators::math::MaxPoolGrad, double>; template class Pool2dGradFunctor< - platform::GPUPlace, paddle::operators::math::avgPoolGrad, double>; + platform::GPUPlace, paddle::operators::math::AvgPoolGrad, double>; template -__global__ void KernelPool3DForward( +__global__ void KernelPool3D( const int nthreads, const T* input_data, T* output_data, const int channels, const int input_depth, const int input_height, const int input_width, const int output_depth, const int output_height, const int output_width, const int ksize_depth, const int ksize_height, const int ksize_width, const int stride_depth, const int stride_height, const int stride_width, const int padding_depth, const int padding_height, const int padding_width, - PoolProcess pool_compute) { - for (int index = blockIdx.x * blockDim.x + threadIdx.x; index < (nthreads); + PoolProcess pool_process) { + for (int index = blockIdx.x * blockDim.x + threadIdx.x; index < nthreads; index += blockDim.x * gridDim.x) { int pw = index % output_width; int ph = (index / output_width) % output_height; @@ -321,25 +325,25 @@ __global__ void KernelPool3DForward( dstart = max(dstart, 0); hstart = max(hstart, 0); wstart = max(wstart, 0); - T ele = pool_compute.initial(); + T ele = pool_process.initial(); input_data += (batch_idx * channels + c) * input_depth * input_height * input_width; for (int d = dstart; d < dend; ++d) { for (int h = hstart; h < hend; ++h) { for (int w = wstart; w < wend; ++w) { - pool_compute.compute( + pool_process.compute( ele, input_data[(d * input_height + h) * input_width + w]); } } } int pool_size = (dend - dstart) * (hend - hstart) * (wend - wstart); - pool_compute.finalize(ele, static_cast(pool_size)); + pool_process.finalize(ele, static_cast(pool_size)); output_data[index] = ele; } } template -__global__ void KernelPool3DBackward( +__global__ void KernelPool3DGrad( const int nthreads, const T* input_data, const T* output_data, const T* output_grad, T* input_grad, const int channels, const int input_depth, const int input_height, const int input_width, @@ -347,8 +351,8 @@ __global__ void KernelPool3DBackward( const int ksize_depth, const int ksize_height, const int ksize_width, const int stride_depth, const int stride_height, const int stride_width, const int padding_depth, const int padding_height, const int padding_width, - PoolProcess pool_compute) { - for (int index = blockIdx.x * blockDim.x + threadIdx.x; index < (nthreads); + PoolProcess pool_process) { + for (int index = blockIdx.x * blockDim.x + threadIdx.x; index < nthreads; index += blockDim.x * gridDim.x) { int offsetW = index % input_width + padding_width; int offsetH = (index / input_width) % input_height + padding_height; @@ -392,7 +396,7 @@ __global__ void KernelPool3DBackward( wstart = max(wstart, 0); int pool_size = (dend - dstart) * (hend - hstart) * (wend - wstart); int output_sub_idx = (pd * output_height + ph) * output_width + pw; - pool_compute.compute(input, output_data[output_sub_idx], + pool_process.compute(input, output_data[output_sub_idx], output_grad[output_sub_idx], gradient, static_cast(1.0 / pool_size)); } @@ -403,7 +407,7 @@ __global__ void KernelPool3DBackward( } template -__global__ void KernelMaxPool3DBackward( +__global__ void KernelMaxPool3DGrad( const int nthreads, const T* input_data, const T* output_data, const T* output_grad, T* input_grad, const int channels, const int input_depth, const int input_height, const int input_width, @@ -412,7 +416,7 @@ __global__ void KernelMaxPool3DBackward( const int stride_depth, const int stride_height, const int stride_width, const int padding_depth, const int padding_height, const int padding_width) { - for (int index = blockIdx.x * blockDim.x + threadIdx.x; index < (nthreads); + for (int index = blockIdx.x * blockDim.x + threadIdx.x; index < nthreads; index += blockDim.x * gridDim.x) { int pw = index % output_width; int ph = (index / output_width) % output_height; @@ -460,7 +464,7 @@ class Pool3dFunctor { void operator()(const platform::DeviceContext& context, const framework::Tensor& input, framework::Tensor& output, std::vector& ksize, std::vector& strides, - std::vector& paddings, PoolProcess pool_compute) { + std::vector& paddings, PoolProcess pool_process) { const int batch_size = input.dims()[0]; const int input_channels = input.dims()[1]; const int input_depth = input.dims()[2]; @@ -489,7 +493,7 @@ class Pool3dFunctor { dim3 threads(1024, 1); dim3 grid(blocks, 1); - KernelPool3DForward< + KernelPool3D< PoolProcess, T><<(context) @@ -498,7 +502,7 @@ class Pool3dFunctor { input_height, input_width, output_depth, output_height, output_width, ksize_depth, ksize_height, ksize_width, stride_depth, stride_height, stride_width, padding_depth, padding_height, padding_width, - pool_compute); + pool_process); } }; @@ -510,7 +514,7 @@ class Pool3dGradFunctor { const framework::Tensor& output, const framework::Tensor& output_grad, std::vector& ksize, std::vector& strides, std::vector& paddings, - PoolProcess pool_compute) { + PoolProcess pool_process) { const int batch_size = input.dims()[0]; const int input_channels = input.dims()[1]; const int input_depth = input.dims()[2]; @@ -541,7 +545,7 @@ class Pool3dGradFunctor { dim3 threads(1024, 1); dim3 grid(blocks, 1); - KernelPool3DBackward< + KernelPool3DGrad< PoolProcess, T><<(context) @@ -550,7 +554,7 @@ class Pool3dGradFunctor { input_channels, input_depth, input_height, input_width, output_depth, output_height, output_width, ksize_depth, ksize_height, ksize_width, stride_depth, stride_height, stride_width, padding_depth, - padding_height, padding_width, pool_compute); + padding_height, padding_width, pool_process); } }; @@ -592,7 +596,7 @@ class MaxPool3dGradFunctor { dim3 threads(1024, 1); dim3 grid(blocks, 1); - KernelMaxPool3DBackward< + KernelMaxPool3DGrad< T><<(context) .stream()>>>( @@ -605,24 +609,26 @@ class MaxPool3dGradFunctor { }; template class MaxPool3dGradFunctor; -// template class MaxPool3dGradFunctor; +// template class MaxPool3dGradFunctor; // The +// 64-bit floating-point version of atomicAdd() is only supported by devices of +// compute capability 6.x and higher. template class Pool3dFunctor, float>; + paddle::operators::math::MaxPool, float>; template class Pool3dFunctor, float>; + paddle::operators::math::AvgPool, float>; template class Pool3dGradFunctor< - platform::GPUPlace, paddle::operators::math::maxPoolGrad, float>; + platform::GPUPlace, paddle::operators::math::MaxPoolGrad, float>; template class Pool3dGradFunctor< - platform::GPUPlace, paddle::operators::math::avgPoolGrad, float>; + platform::GPUPlace, paddle::operators::math::AvgPoolGrad, float>; template class Pool3dFunctor, double>; + paddle::operators::math::MaxPool, double>; template class Pool3dFunctor, double>; + paddle::operators::math::AvgPool, double>; template class Pool3dGradFunctor< - platform::GPUPlace, paddle::operators::math::maxPoolGrad, double>; + platform::GPUPlace, paddle::operators::math::MaxPoolGrad, double>; template class Pool3dGradFunctor< - platform::GPUPlace, paddle::operators::math::avgPoolGrad, double>; + platform::GPUPlace, paddle::operators::math::AvgPoolGrad, double>; } // namespace math } // namespace operators diff --git a/paddle/operators/math/pooling.h b/paddle/operators/math/pooling.h index cf0ab7ecae..d214c68923 100644 --- a/paddle/operators/math/pooling.h +++ b/paddle/operators/math/pooling.h @@ -22,11 +22,10 @@ namespace paddle { namespace operators { namespace math { ////////////////////// -#define FLT_MAX __FLT_MAX__ -///////////////////// +#define FLT_MAX __FLT_MAX__ // template -class maxPool { +class MaxPool { public: DEVICE inline T initial() { return static_cast(-FLT_MAX); } DEVICE inline void compute(T& y, const T& x) { y = y > x ? y : x; } @@ -34,14 +33,14 @@ class maxPool { }; template -class avgPool { +class AvgPool { public: DEVICE inline T initial() { return static_cast(0); } DEVICE inline void compute(T& y, const T& x) { y += x; } DEVICE inline void finalize(T& y, const T& poo_size) { y /= poo_size; } }; template -class maxPoolGrad { +class MaxPoolGrad { public: DEVICE inline void compute(const T& x, const T& y, const T& dy, T& dx, T scale) { @@ -50,7 +49,7 @@ class maxPoolGrad { }; template -class avgPoolGrad { +class AvgPoolGrad { public: DEVICE inline void compute(const T& x, const T& y, const T& dy, T& dx, T scale) { diff --git a/paddle/operators/pool_op.cc b/paddle/operators/pool_op.cc index 6219c6f97c..c29f51f056 100644 --- a/paddle/operators/pool_op.cc +++ b/paddle/operators/pool_op.cc @@ -51,7 +51,7 @@ class PoolOp : public framework::OperatorWithKernel { ksize[i] = static_cast(in_x_dims[i + 2]); } - PADDLE_ENFORCE(in_x_dims.size() - ksize.size() == 2, + PADDLE_ENFORCE(in_x_dims.size() - ksize.size() == 2U, "Input size and Pooling size should be consistent."); PADDLE_ENFORCE(ksize.size() == 2 || ksize.size() == 3, "Pooling size should be 2 elements. or 3 elements."); @@ -79,7 +79,6 @@ class PoolOpGrad : public framework::OperatorWithKernel { "X(Input) of Pooling should not be null."); PADDLE_ENFORCE(ctx->HasOutput(framework::GradVarName("X")), "Input@Grad of Pooling should not be null."); - ctx->SetOutputDim(framework::GradVarName("X"), ctx->GetInputDim("X")); } }; @@ -98,66 +97,36 @@ class Pool2dOpMaker : public framework::OpProtoAndCheckerMaker { "The format of output tensor is also NCHW."); AddAttr("poolingType", - "poolingType of pooling operator." - "str constant equal to 'max' or 'avg'"); + "PoolingType of pooling operator." + "Str constant equal to 'max' or 'avg'.") + .InEnum({"max", "avg"}); AddAttr>( "ksize", "Pooling size(depth, height, width) of pooling operator." - "If globalPooling = true, ksize is ignored and need not be specified."); + "If globalPooling = true, ksize is ignored and need not be " + "specified."); // TODO(Add checker) AddAttr( "globalPooling", - "whether to use the globalPooling." - "int constant equal to false or true" - "default false" + "Whether to use the globalPooling." + "Bool constant equal to false or true." + "Default false." "If globalPooling = true, ksize is ignored and need not be specified.") .SetDefault(false); AddAttr>("strides", - "strides(height, width) of pooling operator." - "default {1,1}") - .SetDefault({1, 1}) - .AddCustomChecker(GreaterThanChecker_pool({0, 0})); + "Strides(height, width) of pooling operator." + "Default {1,1}") + .SetDefault({1, 1}); // TODO(Add checker) AddAttr>("paddings", - "paddings(height, width) of pooling operator." - "default {0,0}") - .SetDefault({0, 0}) - .AddCustomChecker(EqualGreaterThanChecker_pool({0, 0})); + "Paddings(height, width) of pooling operator." + "Default {0,0}.") + .SetDefault({0, 0}); // TODO(Add checker) AddComment(R"DOC( The pooling2d operation calculates the output based on the input, poolingType and ksize, strides, paddings parameters. )DOC"); } - - private: - struct GreaterThanChecker_pool { - public: - explicit GreaterThanChecker_pool(std::vector lower_bound) - : lower_bound_(lower_bound) {} - void operator()(std::vector &value) const { - PADDLE_ENFORCE(value.size() == lower_bound_.size(), "equal check fails."); - for (size_t i = 0; i < value.size(); ++i) { - PADDLE_ENFORCE(value[i] > lower_bound_[i], "larger_than check fails."); - } - } - - private: - std::vector lower_bound_; - }; - - struct EqualGreaterThanChecker_pool { - public: - explicit EqualGreaterThanChecker_pool(std::vector lower_bound) - : lower_bound_(lower_bound) {} - void operator()(std::vector &value) const { - PADDLE_ENFORCE(value.size() == lower_bound_.size(), "equal check fails."); - for (size_t i = 0; i < value.size(); ++i) { - PADDLE_ENFORCE(value[i] >= lower_bound_[i], "larger_than check fails."); - } - } - - private: - std::vector lower_bound_; - }; }; + class Pool3dOpMaker : public framework::OpProtoAndCheckerMaker { public: Pool3dOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) @@ -173,67 +142,36 @@ class Pool3dOpMaker : public framework::OpProtoAndCheckerMaker { "The format of output tensor is also NCDHW."); AddAttr("poolingType", - "poolingType of pooling operator." - "str constant equal to 'max' or 'avg'"); + "PoolingType of pooling operator." + "str constant equal to 'max' or 'avg'.") + .InEnum({"max", "avg"}); AddAttr>( "ksize", - "pooling size(depth, height, width) of pooling operator." - "If globalPooling = true, ksize is ignored and need not be specified."); + "Pooling size(depth, height, width) of pooling operator." + "If globalPooling = true, ksize is ignored and need not be " + "specified."); // TODO(Add checker) AddAttr( "globalPooling", - "whether to use the globalPooling." - "int constant equal to false or true" - "default false" + "Whether to use the globalPooling." + "Bool constant equal to false or true." + "Default false." "If globalPooling = true, ksize is ignored and need not be specified.") .SetDefault(false); AddAttr>( "strides", - "strides(depth, height, width) of pooling operator." - "default {1,1,1}") - .SetDefault({1, 1, 1}) - .AddCustomChecker(GreaterThanChecker_pool({0, 0, 0})); + "Strides(depth, height, width) of pooling operator." + "Default {1,1,1}.") + .SetDefault({1, 1, 1}); // TODO(Add checker) AddAttr>( "paddings", - "paddings(depth, height, width) of pooling operator." - "default {0,0,0}") - .SetDefault({0, 0, 0}) - .AddCustomChecker(EqualGreaterThanChecker_pool({0, 0, 0})); + "Paddings(depth, height, width) of pooling operator." + "Default {0,0,0}.") + .SetDefault({0, 0, 0}); // TODO(Add checker) AddComment(R"DOC( The pooling3d operation calculates the output based on the input, poolingType and ksize, strides, paddings parameters. )DOC"); } - - private: - struct GreaterThanChecker_pool { - public: - explicit GreaterThanChecker_pool(std::vector lower_bound) - : lower_bound_(lower_bound) {} - void operator()(std::vector &value) const { - PADDLE_ENFORCE(value.size() == lower_bound_.size(), "equal check fails."); - for (size_t i = 0; i < value.size(); ++i) { - PADDLE_ENFORCE(value[i] > lower_bound_[i], "larger_than check fails."); - } - } - - private: - std::vector lower_bound_; - }; - - struct EqualGreaterThanChecker_pool { - public: - explicit EqualGreaterThanChecker_pool(std::vector lower_bound) - : lower_bound_(lower_bound) {} - void operator()(std::vector &value) const { - PADDLE_ENFORCE(value.size() == lower_bound_.size(), "equal check fails."); - for (size_t i = 0; i < value.size(); ++i) { - PADDLE_ENFORCE(value[i] >= lower_bound_[i], "larger_than check fails."); - } - } - - private: - std::vector lower_bound_; - }; }; } // namespace operators } // namespace paddle diff --git a/paddle/operators/pool_op.h b/paddle/operators/pool_op.h index 73c9721624..0c246b38ef 100644 --- a/paddle/operators/pool_op.h +++ b/paddle/operators/pool_op.h @@ -31,12 +31,11 @@ class PoolKernel : public framework::OpKernel { const Tensor* in_x = context.Input("X"); Tensor* out = context.Output("Out"); - bool global_pooling = context.Attr("globalPooling"); std::string pooling_type = context.Attr("poolingType"); std::vector ksize = context.Attr>("ksize"); std::vector strides = context.Attr>("strides"); std::vector paddings = context.Attr>("paddings"); - if (global_pooling) { + if (context.Attr("globalPooling")) { for (size_t i = 0; i < ksize.size(); ++i) { ksize[i] = static_cast(in_x->dims()[i + 2]); } @@ -46,17 +45,17 @@ class PoolKernel : public framework::OpKernel { case 2: { if (pooling_type == "max") { paddle::operators::math::Pool2dFunctor< - Place, paddle::operators::math::maxPool, T> + Place, paddle::operators::math::MaxPool, T> pool2d_forward; - paddle::operators::math::maxPool pool_process; + paddle::operators::math::MaxPool pool_process; pool2d_forward(context.device_context(), *in_x, *out, ksize, strides, paddings, pool_process); } else if (pooling_type == "avg") { paddle::operators::math::Pool2dFunctor< - Place, paddle::operators::math::avgPool, T> + Place, paddle::operators::math::AvgPool, T> pool2d_forward; - paddle::operators::math::avgPool pool_process; + paddle::operators::math::AvgPool pool_process; pool2d_forward(context.device_context(), *in_x, *out, ksize, strides, paddings, pool_process); } @@ -64,16 +63,16 @@ class PoolKernel : public framework::OpKernel { case 3: { if (pooling_type == "max") { paddle::operators::math::Pool3dFunctor< - Place, paddle::operators::math::maxPool, T> + Place, paddle::operators::math::MaxPool, T> pool3d_forward; - paddle::operators::math::maxPool pool_process; + paddle::operators::math::MaxPool pool_process; pool3d_forward(context.device_context(), *in_x, *out, ksize, strides, paddings, pool_process); } else if (pooling_type == "avg") { paddle::operators::math::Pool3dFunctor< - Place, paddle::operators::math::avgPool, T> + Place, paddle::operators::math::AvgPool, T> pool3d_forward; - paddle::operators::math::avgPool pool_process; + paddle::operators::math::AvgPool pool_process; pool3d_forward(context.device_context(), *in_x, *out, ksize, strides, paddings, pool_process); } @@ -92,13 +91,12 @@ class PoolGradKernel : public framework::OpKernel { context.Input(framework::GradVarName("Out")); Tensor* in_x_grad = context.Output(framework::GradVarName("X")); - bool global_pooling = context.Attr("globalPooling"); std::string pooling_type = context.Attr("poolingType"); std::vector ksize = context.Attr>("ksize"); std::vector strides = context.Attr>("strides"); std::vector paddings = context.Attr>("paddings"); - if (global_pooling) { + if (context.Attr("globalPooling")) { for (size_t i = 0; i < ksize.size(); ++i) ksize[i] = static_cast(in_x->dims()[i + 2]); } @@ -118,9 +116,9 @@ class PoolGradKernel : public framework::OpKernel { *out_grad, ksize, strides, paddings); } else if (pooling_type == "avg") { paddle::operators::math::Pool2dGradFunctor< - Place, paddle::operators::math::avgPoolGrad, T> + Place, paddle::operators::math::AvgPoolGrad, T> pool2d_backward; - paddle::operators::math::avgPoolGrad pool_process; + paddle::operators::math::AvgPoolGrad pool_process; pool2d_backward(context.device_context(), *in_x, *in_x_grad, *out, *out_grad, ksize, strides, paddings, pool_process); } @@ -133,9 +131,9 @@ class PoolGradKernel : public framework::OpKernel { *out_grad, ksize, strides, paddings); } else if (pooling_type == "avg") { paddle::operators::math::Pool3dGradFunctor< - Place, paddle::operators::math::avgPoolGrad, T> + Place, paddle::operators::math::AvgPoolGrad, T> pool3d_backward; - paddle::operators::math::avgPoolGrad pool_process; + paddle::operators::math::AvgPoolGrad pool_process; pool3d_backward(context.device_context(), *in_x, *in_x_grad, *out, *out_grad, ksize, strides, paddings, pool_process); } From 0a37f2adf0ecab50f979f655e9173a2f13095ce3 Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Fri, 29 Sep 2017 17:20:30 +0800 Subject: [PATCH 044/170] fix typo --- paddle/gserver/tests/test_MKLDNN.cpp | 5 +++-- paddle/scripts/submit_local.sh.in | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/paddle/gserver/tests/test_MKLDNN.cpp b/paddle/gserver/tests/test_MKLDNN.cpp index 857d07df3e..a70b2f17f4 100644 --- a/paddle/gserver/tests/test_MKLDNN.cpp +++ b/paddle/gserver/tests/test_MKLDNN.cpp @@ -215,13 +215,13 @@ struct testActDesc { static void getAddtoConfig(TestConfig& cfg, const testActDesc& pm) { cfg.biasSize = 0; cfg.layerConfig.set_type("addto"); - size_t layerSize = pm.ih * pm.ih * pm.iw; + size_t layerSize = pm.ic * pm.ih * pm.iw; cfg.layerConfig.set_size(layerSize); cfg.inputDefs.push_back({INPUT_DATA, "layer_0", layerSize, 0}); cfg.layerConfig.add_inputs(); } -void testActivation(std::string& actType, const testActDesc& pm) { +void testActivation(std::string actType, const testActDesc& pm) { // TODO(TJ): remove me when paddle support elu activation if (actType == "mkldnn_elu") { return; @@ -240,6 +240,7 @@ TEST(MKLDNNActivation, Activations) { for (auto type : types) { /* bs, c, h, w*/ testActivation(type, {16, 64, 32, 32}); + testActivation(type, {2, 8, 1, 1}); } } diff --git a/paddle/scripts/submit_local.sh.in b/paddle/scripts/submit_local.sh.in index 26f9c0fcd4..5c4b5a2495 100755 --- a/paddle/scripts/submit_local.sh.in +++ b/paddle/scripts/submit_local.sh.in @@ -18,7 +18,7 @@ function version(){ echo "PaddlePaddle @PADDLE_VERSION@, compiled with" echo " with_avx: @WITH_AVX@" echo " with_gpu: @WITH_GPU@" - echo " with_mkldnn: @WITH_MKLDNN" + echo " with_mkldnn: @WITH_MKLDNN@" echo " with_mklml: @WITH_MKLML@" echo " with_double: @WITH_DOUBLE@" echo " with_python: @WITH_PYTHON@" From ff061c25022c0b4ed41ccf3cdbc43c7f04940fe4 Mon Sep 17 00:00:00 2001 From: Luo Tao Date: Fri, 29 Sep 2017 18:05:06 +0800 Subject: [PATCH 045/170] refine "Testing Backward Operators" in new_op_cn/en.md --- doc/howto/dev/new_op_cn.md | 36 +++++++++++------------------------- doc/howto/dev/new_op_en.md | 34 ++++++++++------------------------ 2 files changed, 21 insertions(+), 49 deletions(-) diff --git a/doc/howto/dev/new_op_cn.md b/doc/howto/dev/new_op_cn.md index 264b998f50..9d3d02ffc3 100644 --- a/doc/howto/dev/new_op_cn.md +++ b/doc/howto/dev/new_op_cn.md @@ -285,41 +285,27 @@ class TestMulGradOp(GradientChecker): 'Y': np.random.random((84, 100)).astype("float32") } - def test_cpu_gpu_compare(self): - self.compare_grad(self.op, self.inputs) - - def test_normal(self): + def test_check_grad_normal(self): # mul op will enlarge the relative error - self.check_grad( - self.op, self.inputs, ["X", "Y"], "Out", max_relative_error=0.5) + self.check_grad(['X', 'Y'], 'Out', max_relative_error=0.5) - def test_ignore_x(self): + def test_check_grad_ingore_x(self): self.check_grad( - self.op, - self.inputs, ["Y"], - "Out", - max_relative_error=0.5, - no_grad_set={"X"}) + ['Y'], 'Out', max_relative_error=0.5, no_grad_set=set("X")) - def test_ignore_y(self): + def test_check_grad_ingore_y(self): self.check_grad( - self.op, - self.inputs, ["X"], - "Out", - max_relative_error=0.5, - no_grad_set={"Y"}) + ['X'], 'Out', max_relative_error=0.5, no_grad_set=set('Y')) ``` 下面解释代码中一些关键的地方: - 调用`create_op("mul")`创建反向Op对应的前向Op。 -- 调用`compare_grad`函数对比CPU、GPU计算结果。 -- `test_normal`中调用`check_grad`使用数值法检测梯度正确性和稳定性。 - - 第一个参数`self.op` : 前向Op。 - - 第二个参数`self.inputs` : 输入词典，词典的Key和`ProtoMaker`定义保持一致。 - - 第三个参数`["X", "Y"]` : 指定对输入变量`X`、`Y`做梯度检测。 - - 第四个参数`"Out"` : 指定前向网络最终的输出目标变量`Out` -- `test_ignore_x`和`test_ignore_y`分支用来测试只需要计算一个输入梯度的情况。 +- `test_check_grad_normal`中调用`check_grad`使用数值法检测梯度正确性和稳定性。 + - 第一个参数`["X", "Y"]` : 指定对输入变量`X`、`Y`做梯度检测。 + - 第二个参数`"Out"` : 指定前向网络最终的输出目标变量`Out`。 + - 第三个参数`max_relative_error`：指定检测梯度时能容忍的最大错误值。 +- `test_check_grad_ingore_x`和`test_check_grad_ingore_y`分支用来测试只需要计算一个输入梯度的情况。 ### 编译和执行单元测试 diff --git a/doc/howto/dev/new_op_en.md b/doc/howto/dev/new_op_en.md index bad1dbc1de..57ff7caad1 100644 --- a/doc/howto/dev/new_op_en.md +++ b/doc/howto/dev/new_op_en.md @@ -293,41 +293,27 @@ class TestMulGradOp(GradientChecker): 'Y': np.random.random((84, 100)).astype("float32") } - def test_cpu_gpu_compare(self): - self.compare_grad(self.op, self.inputs) - - def test_normal(self): + def test_check_grad_normal(self): # mul op will enlarge the relative error - self.check_grad( - self.op, self.inputs, ["X", "Y"], "Out", max_relative_error=0.5) + self.check_grad(['X', 'Y'], 'Out', max_relative_error=0.5) - def test_ignore_x(self): + def test_check_grad_ingore_x(self): self.check_grad( - self.op, - self.inputs, ["Y"], - "Out", - max_relative_error=0.5, - no_grad_set={"X"}) + ['Y'], 'Out', max_relative_error=0.5, no_grad_set=set("X")) - def test_ignore_y(self): + def test_check_grad_ingore_y(self): self.check_grad( - self.op, - self.inputs, ["X"], - "Out", - max_relative_error=0.5, - no_grad_set={"Y"}) + ['X'], 'Out', max_relative_error=0.5, no_grad_set=set('Y')) ``` Some key points in the code above include: - `create_op("mul")` creates the backward operator's corresponding forward operator. -- `compare_grad` compares results between utilizing the CPU and the GPU. - `test_normal` calls `check_grad` to validate scaling tests' correctness and stability through numeric methods. - - The first variable `self.op` denotes the forward operator. - - The second variable `self.inputs` denotes the input dictionary, which has its key value identical to its `ProtoMaker` definitions. - - The third variable `["X", "Y"]` appoints `X` and `Y` to be scale tested. - - The fourth variable `"Out"` points to the network's final output target `Out`. -- `test_ignore_x` and `test_ignore_y`branches test the cases where there is only one scaling input. + - The first variable `["X", "Y"]` appoints `X` and `Y` to be scale tested. + - The second variable `"Out"` points to the network's final output target `Out`. + - The third variable `max_relative_error` points to the maximum relative tolerance error during scaling tests. +- `test_check_grad_ingore_x` and `test_check_grad_ingore_y`branches test the cases where there is only one scaling input. ### Compiling and Running From 46641c6361ebcf4c9fab79a70c4251ce4329a0f7 Mon Sep 17 00:00:00 2001 From: guosheng Date: Fri, 29 Sep 2017 19:16:33 +0800 Subject: [PATCH 046/170] Fix infer when input is empty in v2/inference.py --- python/paddle/v2/inference.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/python/paddle/v2/inference.py b/python/paddle/v2/inference.py index e80456d9bb..9148cb56cf 100644 --- a/python/paddle/v2/inference.py +++ b/python/paddle/v2/inference.py @@ -96,6 +96,9 @@ class Inference(object): for i, item in enumerate(result): retv[i].append(item) + if retv == None: + return [] + if flatten_result: retv = [numpy.concatenate(out) for out in retv] From 762a99cc064564102c6d8a6f6e68494ab74a3acd Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Thu, 28 Sep 2017 19:37:42 -0700 Subject: [PATCH 047/170] Remove add_op since it can be replaced by sum_op --- paddle/framework/CMakeLists.txt | 2 +- paddle/framework/grad_op_builder_test.cc | 25 ++++--- paddle/operators/add_op.cc | 68 ------------------- paddle/operators/add_op.cu | 18 ----- paddle/operators/add_op.h | 48 ------------- paddle/operators/sum_op.cc | 6 +- .../paddle/v2/framework/tests/test_add_op.py | 20 ------ 7 files changed, 21 insertions(+), 166 deletions(-) delete mode 100644 paddle/operators/add_op.cc delete mode 100644 paddle/operators/add_op.cu delete mode 100644 paddle/operators/add_op.h delete mode 100644 python/paddle/v2/framework/tests/test_add_op.py diff --git a/paddle/framework/CMakeLists.txt b/paddle/framework/CMakeLists.txt index 8a5d8532bb..c0ad888b70 100644 --- a/paddle/framework/CMakeLists.txt +++ b/paddle/framework/CMakeLists.txt @@ -29,7 +29,7 @@ cc_test(operator_test SRCS operator_test.cc DEPS operator op_registry) cc_library(grad_op_builder SRCS grad_op_builder.cc DEPS operator proto_desc) cc_library(op_registry SRCS op_registry.cc DEPS grad_op_builder op_proto_maker op_info) cc_test(op_registry_test SRCS op_registry_test.cc DEPS op_registry) -cc_test(grad_op_builder_test SRCS grad_op_builder_test.cc DEPS grad_op_builder op_registry add_op) +cc_test(grad_op_builder_test SRCS grad_op_builder_test.cc DEPS grad_op_builder op_registry sum_op) py_proto_compile(framework_py_proto SRCS framework.proto) # Generate an empty __init__.py to make framework_py_proto as a valid python module. diff --git a/paddle/framework/grad_op_builder_test.cc b/paddle/framework/grad_op_builder_test.cc index d09892f81b..55c5fa420e 100644 --- a/paddle/framework/grad_op_builder_test.cc +++ b/paddle/framework/grad_op_builder_test.cc @@ -3,7 +3,7 @@ #include "paddle/framework/op_registry.h" #include "paddle/framework/operator.h" -USE_OP(add); +USE_OP(sum); namespace paddle { namespace framework { @@ -41,17 +41,24 @@ namespace f = paddle::framework; TEST(GradOpBuilder, AddTwo) { std::shared_ptr add_op(f::OpRegistry::CreateOp( - "add", {{"X", {"x"}}, {"Y", {"y"}}}, {{"Out", {"out"}}}, {})); + "sum", {{"X", {"x", "y"}}}, {{"Out", {"out"}}}, {})); std::shared_ptr grad_add_op = f::OpRegistry::CreateGradOp(*add_op); - EXPECT_EQ(grad_add_op->Inputs().size(), 4UL); - EXPECT_EQ(grad_add_op->Outputs().size(), 2UL); - EXPECT_EQ(grad_add_op->Input("X"), "x"); - EXPECT_EQ(grad_add_op->Input("Y"), "y"); - EXPECT_EQ(grad_add_op->Input("Out"), "out"); + + EXPECT_EQ(grad_add_op->Inputs().size(), 1UL); + EXPECT_EQ(grad_add_op->Outputs().size(), 1UL); EXPECT_EQ(grad_add_op->Input(f::GradVarName("Out")), f::GradVarName("out")); - EXPECT_EQ(grad_add_op->Output(f::GradVarName("X")), f::GradVarName("x")); - EXPECT_EQ(grad_add_op->Output(f::GradVarName("Y")), f::GradVarName("y")); + auto &outputs = grad_add_op->Outputs(f::GradVarName("X")); + EXPECT_EQ(2UL, outputs.size()); + auto in_output = [&outputs](const std::string &name) { + for (auto &output_name : outputs) { + if (output_name == name) return true; + } + return false; + }; + + EXPECT_TRUE(in_output(f::GradVarName("x"))); + EXPECT_TRUE(in_output(f::GradVarName("y"))); } REGISTER_OP(mult_io, f::NOP, f::MutiInOutOpMaker, mult_io_grad, f::NOP); diff --git a/paddle/operators/add_op.cc b/paddle/operators/add_op.cc deleted file mode 100644 index 3914d13230..0000000000 --- a/paddle/operators/add_op.cc +++ /dev/null @@ -1,68 +0,0 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ - -#include "paddle/operators/add_op.h" - -namespace paddle { -namespace operators { - -class AddOp : public framework::OperatorWithKernel { - public: - using framework::OperatorWithKernel::OperatorWithKernel; - - protected: - void InferShape(framework::InferShapeContextBase* ctx) const override { - PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) of AddOp should not be null."); - PADDLE_ENFORCE(ctx->HasInput("Y"), "Input(Y) of AddOp should not be null."); - PADDLE_ENFORCE(ctx->HasOutput("Out"), - "Output(Out) of AddOp should not be null."); - - auto x_dims = ctx->GetInputDim("X"); - auto y_dims = ctx->GetInputDim("Y"); - PADDLE_ENFORCE_EQ(x_dims, y_dims, - "Two input of Add Op's dimension must be same."); - ctx->SetOutputDim("Out", x_dims); - } -}; - -class AddOpMaker : public framework::OpProtoAndCheckerMaker { - public: - AddOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) - : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("X", "The first input of add op"); - AddInput("Y", "The second input of add op"); - AddOutput("Out", "The output of add op"); - AddComment(R"DOC( -Two Element Add Operator. - -The equation is: Out = X + Y -)DOC"); - } -}; - -class AddOpGrad : public framework::OperatorWithKernel { - public: - using framework::OperatorWithKernel::OperatorWithKernel; - - protected: - void InferShape(framework::InferShapeContextBase* ctx) const override {} -}; - -} // namespace operators -} // namespace paddle - -namespace ops = paddle::operators; -REGISTER_OP(add, ops::AddOp, ops::AddOpMaker, add_grad, ops::AddOpGrad); - -REGISTER_OP_CPU_KERNEL(add, ops::AddKernel); diff --git a/paddle/operators/add_op.cu b/paddle/operators/add_op.cu deleted file mode 100644 index d9c6d20a6c..0000000000 --- a/paddle/operators/add_op.cu +++ /dev/null @@ -1,18 +0,0 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ - -#include "paddle/operators/add_op.h" - -namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL(add, ops::AddKernel); diff --git a/paddle/operators/add_op.h b/paddle/operators/add_op.h deleted file mode 100644 index 75163032a1..0000000000 --- a/paddle/operators/add_op.h +++ /dev/null @@ -1,48 +0,0 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ - -#pragma once -#include "paddle/framework/eigen.h" -#include "paddle/framework/op_registry.h" - -namespace paddle { -namespace operators { - -using Tensor = framework::Tensor; -template -using EigenVector = framework::EigenVector; - -template -class AddKernel : public framework::OpKernel { - public: - void Compute(const framework::ExecutionContext& context) const override { - auto* input0 = context.Input("X"); - auto* input1 = context.Input("Y"); - auto* output = context.Output("Out"); - - output->mutable_data(context.GetPlace()); - - auto X = EigenVector::Flatten(*input0); - auto Y = EigenVector::Flatten(*input1); - auto Z = EigenVector::Flatten(*output); - - auto place = context.GetEigenDevice(); - - Z.device(place) = X + Y; - } -}; - -} // namespace operators -} // namespace paddle diff --git a/paddle/operators/sum_op.cc b/paddle/operators/sum_op.cc index 8f62a9f4db..5d76313aeb 100644 --- a/paddle/operators/sum_op.cc +++ b/paddle/operators/sum_op.cc @@ -43,8 +43,10 @@ class SumOpMaker : public framework::OpProtoAndCheckerMaker { public: SumOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("X", "the input tensors of sum operator.").AsDuplicable(); - AddOutput("Out", "the output tensor of sum operator."); + AddInput("X", "the input tensors of sum operator.") + .AsDuplicable() + .NotInGradient(); + AddOutput("Out", "the output tensor of sum operator.").NotInGradient(); AddComment(R"DOC( Sum the input tensors. diff --git a/python/paddle/v2/framework/tests/test_add_op.py b/python/paddle/v2/framework/tests/test_add_op.py deleted file mode 100644 index 3ca34d9b9f..0000000000 --- a/python/paddle/v2/framework/tests/test_add_op.py +++ /dev/null @@ -1,20 +0,0 @@ -import unittest -import numpy as np -from op_test import OpTest - - -class TestAddOp(OpTest): - def setUp(self): - self.op_type = "add" - self.inputs = { - 'X': np.random.random((102, 105)).astype("float32"), - 'Y': np.random.random((102, 105)).astype("float32") - } - self.outputs = {'Out': self.inputs['X'] + self.inputs['Y']} - - def test_check_output(self): - self.check_output() - - -if __name__ == "__main__": - unittest.main() From 37c9bc609c807d200e08e6dc11492887fa7673d9 Mon Sep 17 00:00:00 2001 From: Kavya Srinet Date: Fri, 29 Sep 2017 10:52:50 -0700 Subject: [PATCH 048/170] Few more edits --- doc/design/refactorization.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/design/refactorization.md b/doc/design/refactorization.md index 715f209d3f..629422e774 100644 --- a/doc/design/refactorization.md +++ b/doc/design/refactorization.md @@ -17,7 +17,7 @@ The goals of refactoring include: 1. A graph is composed of *variables* and *operators*. -1. The description of graphs must be capable of being serialized/deserialized, so that +1. The description of graphs must be capable of being serialized/deserialized, so that: 1. It can to be sent to the cloud for distributed execution, and 1. It can be sent to clients for mobile or enterprise deployment. From 6164b8986e473daa00e1a71332cc472e21627f55 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Fri, 29 Sep 2017 12:48:40 -0700 Subject: [PATCH 049/170] Fix CI --- .../framework/tests/test_gradient_checker.py | 46 ------------------- python/paddle/v2/framework/tests/test_net.py | 4 +- .../v2/framework/tests/test_operator.py | 4 +- 3 files changed, 4 insertions(+), 50 deletions(-) delete mode 100644 python/paddle/v2/framework/tests/test_gradient_checker.py diff --git a/python/paddle/v2/framework/tests/test_gradient_checker.py b/python/paddle/v2/framework/tests/test_gradient_checker.py deleted file mode 100644 index 85117bf960..0000000000 --- a/python/paddle/v2/framework/tests/test_gradient_checker.py +++ /dev/null @@ -1,46 +0,0 @@ -import unittest -import numpy as np -import paddle.v2.framework.core as core -from op_test import get_numeric_gradient -from op_test import create_op - - -class GetNumericGradientTest(unittest.TestCase): - def test_add_op(self): - x = np.random.random((10, 1)).astype("float32") - y = np.random.random((10, 1)).astype("float32") - z = x + y - scope = core.Scope() - add_op = create_op(scope, "add", {'X': x, 'Y': y}, {'Out': z}, dict()) - arr = get_numeric_gradient(scope, add_op, {'X': x, - 'Y': y}, 'X', ['Out']) - self.assertAlmostEqual(arr.mean(), 1.0, delta=1e-4) - - def test_softmax_op(self): - def stable_softmax(x): - """Compute the softmax of vector x in a numerically stable way.""" - shiftx = x - np.max(x) - exps = np.exp(shiftx) - return exps / np.sum(exps) - - def label_softmax_grad(Y, dY): - dX = Y * 0.0 - for i in range(Y.shape[0]): - d = np.dot(Y[i, :], dY[i, :]) - dX[i, :] = Y[i, :] * (dY[i, :] - d) - return dX - - X = np.random.random((2, 2)).astype("float32") - Y = np.apply_along_axis(stable_softmax, 1, X) - dY = np.ones(Y.shape) - dX = label_softmax_grad(Y, dY) - - scope = core.Scope() - softmax_op = create_op(scope, "softmax", {"X": X}, {"Y": Y}, dict()) - - arr = get_numeric_gradient(scope, softmax_op, {"X": X}, "X", "Y") - np.testing.assert_almost_equal(arr, dX, decimal=1e-2) - - -if __name__ == "__main__": - unittest.main() diff --git a/python/paddle/v2/framework/tests/test_net.py b/python/paddle/v2/framework/tests/test_net.py index 50cfb855f2..8503257feb 100644 --- a/python/paddle/v2/framework/tests/test_net.py +++ b/python/paddle/v2/framework/tests/test_net.py @@ -15,7 +15,7 @@ def fc(X, W, Y): class TestNet(unittest.TestCase): def test_net_all(self): net = core.Net.create() - op1 = Operator("add", X="X", Y="Y", Out="Out") + op1 = Operator("sum", X=["X", "Y"], Out="Out") net.append_op(op1) net2 = core.Net.create() @@ -26,7 +26,7 @@ class TestNet(unittest.TestCase): expected = ''' Op(plain_net), inputs:{all[W, X, Y]}, outputs:{all[Out, fc.out, pre_activation]}. - Op(add), inputs:{X[X], Y[Y]}, outputs:{Out[Out]}. + Op(sum), inputs:{X[X, Y]}, outputs:{Out[Out]}. Op(plain_net), inputs:{all[W, X]}, outputs:{all[fc.out, pre_activation]}. Op(plain_net), inputs:{all[W, X]}, outputs:{all[fc.out, pre_activation]}. Op(mul), inputs:{X[X], Y[W]}, outputs:{Out[pre_activation]}. diff --git a/python/paddle/v2/framework/tests/test_operator.py b/python/paddle/v2/framework/tests/test_operator.py index 040556322d..98f6b2f5ee 100644 --- a/python/paddle/v2/framework/tests/test_operator.py +++ b/python/paddle/v2/framework/tests/test_operator.py @@ -193,10 +193,10 @@ class TestOpDescCreationMethod(unittest.TestCase): class TestOpCreations(unittest.TestCase): def test_all(self): - add_op = op.Operator("add", X="a", Y="b", Out="z") + add_op = op.Operator("sum", X=["a", "b"], Out="z") self.assertIsNotNone(add_op) # Invoke C++ DebugString() - self.assertEqual('Op(add), inputs:{X[a], Y[b]}, outputs:{Out[z]}.', + self.assertEqual('Op(sum), inputs:{X[a, b]}, outputs:{Out[z]}.', str(add_op)) From 12713e92e4a28fa7de762d9be76125efdace7ae2 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Fri, 29 Sep 2017 13:49:18 -0700 Subject: [PATCH 050/170] Design doc of compile time register gradient operators --- doc/design/register_grad_op.md | 63 ++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 doc/design/register_grad_op.md diff --git a/doc/design/register_grad_op.md b/doc/design/register_grad_op.md new file mode 100644 index 0000000000..1c961a6158 --- /dev/null +++ b/doc/design/register_grad_op.md @@ -0,0 +1,63 @@ +# Design Doc: Register Gradient Operator + +## Problem + +Since we separate users program in two stages, compile time and runtime, we should record and look up the mapping relationship between an operator and its gradient operators when compile. However, we register this relationship in runtime by these `OpInfo` fields. + +```cpp +struct OpInfo { + std::function creator_; + std::string grad_op_type_; + ... +}; +``` + +OpInfos store in a association map which key is the operator type. The `grad_op_type` indicate associated gradient operator type. Operator can create gradient operator by `OpInfo::creator_` of gradient. The pseudo code is + +```cpp +map OpInfoMap; + +OperatorBase* CreateGradientOperator(const OperatorBase& op) { + return OpInfoMap.at(op.Type()).creator_(...); +} +``` + +At the same time, an operator's gradient operator could be composed of many forward operators. For example, the gradient operator of `minus_op` could consist of an `identity` operator and a `scale` operator. To compose a gradient operator by forwarding operators could: 1) Reuse forwarding operator; 2) Calculate second derivative, third derivative, etc. + +We use `NetOp` to represent a composed operator since the `NetOp` is `vector`. However, `NetOp` is also a runtime concept. We should provide a mechanism to compose operators as a gradient operator. + +In conclusion, the problem that we want to resolve in this design doc is to register the mapping relationship between the forward operator and its gradient operators during compile time. + + +## Solution + +The mapping relationship between an operator and its gradient operators is a function. The interface of that function is: + +```cpp +// (OpDesc) --> vector +using GradOpDescMaker = std::function(const OpDesc&)>; +``` + +The function take a `OpDesc` of the forward operator and return one or many gradient operator descriptions. + +The `GradOpDescMaker` will be registered in `OpInfo`, to replace `grad_op_type_` field. The `OpInfo` should be + +```cpp +struct OpInfo { + GradOpDescMaker grad_op_maker_; + ... +}; +``` + +The `grad_op_maker_ ` is `nullptr` if the operator does not have associated gradient operators. + +We should chagne register macros at the same time. In the current solution, there is no difference between forwarding operators and backward operators. So `REGISTER_OP` just register one operator. If the `REGISTER_OP` contains `OpProtoAndCheckerMaker` and `GradOpDescMaker ƒ`, we just list them in the same macro. It can be done by a macro contains `__VA_ARGS__`. + +The user interface should be + +```cpp +vector SumOpGradMakerƒ(OpDesc) {...} +REGISTER_OP(sum, SumOp, SumOpProtoAndCheckerMaker, SumOpGradMaker); +// Developers can still manually implement gradient operator. +REGISTER_OP(sum_grad, SumGradOp); +``` From 5ee709380b5e03f334d5eff485cc6e37ac32e432 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Fri, 29 Sep 2017 13:53:44 -0700 Subject: [PATCH 051/170] Not change macro before --- doc/design/register_grad_op.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/doc/design/register_grad_op.md b/doc/design/register_grad_op.md index 1c961a6158..a793cfe96b 100644 --- a/doc/design/register_grad_op.md +++ b/doc/design/register_grad_op.md @@ -51,13 +51,19 @@ struct OpInfo { The `grad_op_maker_ ` is `nullptr` if the operator does not have associated gradient operators. -We should chagne register macros at the same time. In the current solution, there is no difference between forwarding operators and backward operators. So `REGISTER_OP` just register one operator. If the `REGISTER_OP` contains `OpProtoAndCheckerMaker` and `GradOpDescMaker ƒ`, we just list them in the same macro. It can be done by a macro contains `__VA_ARGS__`. +We should chagne register macros at the same time. In the current solution, there is no difference between forwarding operators and backward operators. So `REGISTER_OP` just register one operator. If the `REGISTER_OPERATOR ` contains `OpProtoAndCheckerMaker` and `GradOpDescMaker`, we just list them in the same macro. It can be done by a macro contains `__VA_ARGS__`. The user interface should be ```cpp vector SumOpGradMakerƒ(OpDesc) {...} -REGISTER_OP(sum, SumOp, SumOpProtoAndCheckerMaker, SumOpGradMaker); +REGISTER_OPERATOR(sum, SumOp, SumOpProtoAndCheckerMaker, SumOpGradMaker); // Developers can still manually implement gradient operator. -REGISTER_OP(sum_grad, SumGradOp); +REGISTER_OPERATOR(sum_grad, SumGradOp); +``` + +The interface of current `REGISTER_OP` macro could not be changed. In `REGISTER_OP`, it will invoke `REGISTER_OPERATOR` two times and generate GradOpDescMaker inside. + +```cpp +REGISTER_OP(sum, SumOp, SumOpProtoAndCheckerMaker, sum_grad, SumGradOp); ``` From 56f60ee554696d42606fc6765dd2002cebafabe8 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Fri, 29 Sep 2017 17:09:12 -0700 Subject: [PATCH 052/170] Update example --- doc/design/register_grad_op.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/design/register_grad_op.md b/doc/design/register_grad_op.md index a793cfe96b..02c4d554d0 100644 --- a/doc/design/register_grad_op.md +++ b/doc/design/register_grad_op.md @@ -56,14 +56,14 @@ We should chagne register macros at the same time. In the current solution, ther The user interface should be ```cpp -vector SumOpGradMakerƒ(OpDesc) {...} -REGISTER_OPERATOR(sum, SumOp, SumOpProtoAndCheckerMaker, SumOpGradMaker); +vector MinusOpGradMaker(OpDesc) {...} +REGISTER_OPERATOR(minus, MinusOp, MinusOpProtoAndCheckerMaker, SumOpGradMaker); // Developers can still manually implement gradient operator. -REGISTER_OPERATOR(sum_grad, SumGradOp); +REGISTER_OPERATOR(minus_grad, MinusGradOp); ``` The interface of current `REGISTER_OP` macro could not be changed. In `REGISTER_OP`, it will invoke `REGISTER_OPERATOR` two times and generate GradOpDescMaker inside. ```cpp -REGISTER_OP(sum, SumOp, SumOpProtoAndCheckerMaker, sum_grad, SumGradOp); +REGISTER_OP(minus, MinusOp, MinusOpProtoAndCheckerMaker, minus_grad, MinusGradOp); ``` From 7fe0297e6451db4f18df81a8716414acca819529 Mon Sep 17 00:00:00 2001 From: Qiao Longfei Date: Fri, 29 Sep 2017 18:38:36 -0700 Subject: [PATCH 053/170] remove Runtime InferShape for cond op (#4518) * init remove cond_op infershape * optimize code * add PrepareDataForSubnet and MergeDataFromSubnet --- paddle/operators/cond_op.cc | 242 +++++++++--------- paddle/operators/cond_op.h | 33 +-- paddle/operators/scatter.h | 4 - .../paddle/v2/framework/tests/test_cond_op.py | 3 - 4 files changed, 140 insertions(+), 142 deletions(-) diff --git a/paddle/operators/cond_op.cc b/paddle/operators/cond_op.cc index aaffa6661f..db20b69f3f 100644 --- a/paddle/operators/cond_op.cc +++ b/paddle/operators/cond_op.cc @@ -14,12 +14,7 @@ limitations under the License. */ #include "paddle/operators/cond_op.h" -#include -#include - -#include "paddle/framework/op_registry.h" #include "paddle/operators/gather.h" -#include "paddle/operators/net_op.h" #include "paddle/operators/scatter.h" namespace paddle { @@ -31,142 +26,104 @@ using Tensor = framework::Tensor; using LoDTensor = framework::LoDTensor; using DDim = framework::DDim; -void CondOp::CreateScope(const Scope& scope) const { +framework::Scope& CondOp::AddSubScope(const Scope& scope) const { auto sub_scopes_var = scope.FindVar("SubScopes"); PADDLE_ENFORCE_NOT_NULL(sub_scopes_var, "Output(SubScopes) of CondOp should not be null."); auto sub_scopes = sub_scopes_var->GetMutable>(); auto& sub_scope = scope.NewScope(); sub_scopes->push_back(&sub_scope); + return sub_scope; } -void CondOp::CreateIndexTensor(const Scope& scope) const { +std::vector& CondOp::GetSubScopes( + const framework::Scope& scope) const { + auto sub_scopes_var = scope.FindVar("SubScopes"); + PADDLE_ENFORCE_NOT_NULL(sub_scopes_var, + "Output(SubScopes) of CondOp should not be null."); + return *sub_scopes_var->GetMutable>(); +} + +LoDTensor& CondOp::AddIndexTensor(const Scope& scope) const { auto index_tensors_var = scope.FindVar("IndexTensors"); PADDLE_ENFORCE_NOT_NULL(index_tensors_var, "Output(IndexTensors) of CondOp should not be null."); auto& index_tensors = *index_tensors_var->GetMutable>(); index_tensors.push_back(LoDTensor()); + return index_tensors.back(); } -void CondOp::InferShape(const Scope& scope) const { - auto sub_scopes_var = scope.FindVar("SubScopes"); - PADDLE_ENFORCE_NOT_NULL(sub_scopes_var, - "Output(SubScopes) of CondOp should not be null."); - auto& sub_scopes = *sub_scopes_var->GetMutable>(); - - for (int i = 0; i < 2; ++i) { - // Create two sub scopes for true and false branches - // sub_scopes[0] for the true branch and sub_scopes[1] for the false - // branch - CreateScope(scope); - - // Create two tensors for true and false indices - // index_tensors[0] for the true branch and index_tensors[1] for the false - // branch - CreateIndexTensor(scope); - - PADDLE_ENFORCE(!Inputs("Xs").empty(), - "Inputs(Xs) of CondOp can't be empty."); - for (auto& input : Inputs("Xs")) { - // Create a new tensor in sub-scope for input-type tensor - Variable* v = sub_scopes[i]->NewVar(input); - LoDTensor* sub_input = v->GetMutable(); - sub_input->Resize(scope.FindVar(input)->GetMutable()->dims()); - } - - for (auto& output : (*sub_net_op_[i]).Outputs()) { - for (auto& var_name : output.second) { - sub_scopes[i]->NewVar(var_name); - } - } - - // each net calls InferShape - // sub_net_op_[i]->InferShape(*sub_scopes[i]); - } - - for (auto& output : Outputs("Outs")) { - LoDTensor* tensor_t_out = - sub_scopes[0]->FindVar(output)->GetMutable(); - PADDLE_ENFORCE_NOT_NULL(tensor_t_out, "True output should not be NULL"); - LoDTensor* tensor_f_out = - sub_scopes[1]->FindVar(output)->GetMutable(); - PADDLE_ENFORCE_NOT_NULL(tensor_f_out, "False output should not be NULL"); - - auto* tensor_out_var = scope.FindVar(output); - PADDLE_ENFORCE_NOT_NULL(tensor_out_var, "Output not found"); - LoDTensor* tensor_out = tensor_out_var->GetMutable(); - PADDLE_ENFORCE_NOT_NULL(tensor_t_out, - "True output tensor should not be NULL"); - - // check output size should be same - PADDLE_ENFORCE_EQ(tensor_t_out->dims(), tensor_f_out->dims(), - "Outputs not of the same shape"); - tensor_out->Resize(tensor_t_out->dims()); - // tensor_out->mutable_data(tensor_out->dims(), - // platform::CPUPlace()); - tensor_out->mutable_data(platform::CPUPlace()); - } -} - -void CondOp::Run(const Scope& scope, - const platform::DeviceContext& dev_ctx) const { - auto* sub_scopes_var = scope.FindVar("SubScopes"); - PADDLE_ENFORCE_NOT_NULL(sub_scopes_var, - "Output(SubScopes) of CondOp should not be null."); - auto sub_scopes = sub_scopes_var->Get>(); +std::vector& CondOp::GetIndexTensors( + const framework::Scope& scope) const { auto* index_tensors_var = scope.FindVar("IndexTensors"); PADDLE_ENFORCE_NOT_NULL(index_tensors_var, "Output(IndexTensors) of CondOp should not be null."); - auto index_tensors = index_tensors_var->Get>(); + return *index_tensors_var->GetMutable>(); +} - std::string cond_name = Input("Cond"); - Variable* cond_var = scope.FindVar(cond_name); +void CondOp::PrepareDataForSubnet( + const framework::Scope& scope, + const platform::DeviceContext& dev_ctx) const { + PADDLE_ENFORCE(!Inputs("Xs").empty(), "Inputs(Xs) of CondOp can't be empty."); + + for (int i = 0; i < BRANCH_NUM; ++i) { + // Create two sub scopes for true and false branches + // sub_scopes[0] for the true branch + // sub_scopes[1] for the false branch + AddSubScope(scope); + // Create two tensors for true and false indices: + // index_tensors[0] for the true branch + // index_tensors[1] for the false branch + AddIndexTensor(scope); + } + + Variable* cond_var = scope.FindVar(Input("Cond")); PADDLE_ENFORCE_NOT_NULL(cond_var, "Input(Cond) of CondOp should not be null."); const LoDTensor* cond = cond_var->GetMutable(); - // Step 1: get the true/false index at runtime - // index_[0]: vector, contains all index for cond[i] == true - // index_[1]: vector, contains all index for cond[i] == false - for (int i = 0; i < 2; ++i) index_[i].clear(); + // get the true/false index at runtime according to cond tensor + // index_vectors[0]: vector, contains all index for cond[i] == true + // index_vectors[1]: vector, contains all index for cond[i] == false + std::vector> index_vectors; + index_vectors.resize(BRANCH_NUM); const int* cond_data = cond->data(); for (int i = 0; i < cond->dims()[0]; ++i) { if (cond_data[i]) - index_[0].push_back(i); + index_vectors[TRUE_BRANCH].push_back(i); else - index_[1].push_back(i); + index_vectors[FALSE_BRANCH].push_back(i); } - // put index_[0] and index_[1] into two tensors: - // index_tensor_[0] and index_tensor_[1] - DDim dim = paddle::framework::make_ddim({0}); - for (int i = 0; i < 2; ++i) { - dim[0] = index_[i].size(); - int* tmp_ptr = + // put index_vectors[0] and index_vectors[1] into two tensors: + // index_tensors[0] and index_tensors[1] + std::vector& index_tensors = GetIndexTensors(scope); + std::vector& sub_scopes = GetSubScopes(scope); + + for (int i = 0; i < BRANCH_NUM; ++i) { + DDim dim = {static_cast(index_vectors[i].size())}; + int* index_tensor_data_ptr = index_tensors[i].mutable_data(dim, platform::CPUPlace()); - index_tensors[i].Resize(dim); - memcpy(tmp_ptr, index_[i].data(), dim[0] * sizeof(int)); + memcpy(index_tensor_data_ptr, index_vectors[i].data(), + dim[0] * sizeof(int)); } - // Step 2: collect data by calling gather - for (int i = 0; i < 2; ++i) { - // i= 0/i for True and False branches respectively - for (auto& input : Inputs("Xs")) { - // find Tensor - Variable* v = scope.FindVar(input); - PADDLE_ENFORCE_NOT_NULL(v); - LoDTensor* tensor_parent = v->GetMutable(); + // create input in subscopes according to index_vectors + for (auto& input : Inputs("Xs")) { + Variable* var_parent = scope.FindVar(input); + PADDLE_ENFORCE_NOT_NULL(var_parent); + const auto* tensor_parent = &var_parent->Get(); - v = sub_scopes[i]->FindVar(input); - PADDLE_ENFORCE_NOT_NULL(v); - LoDTensor* tensor_child = v->GetMutable(); + for (int i = 0; i < BRANCH_NUM; ++i) { + Variable* var_child = sub_scopes[i]->FindVar(input); + PADDLE_ENFORCE_NOT_NULL(var_child); + auto* tensor_child = var_child->GetMutable(); // Resize child - DDim dim = tensor_child->dims(); - dim[0] = index_[i].size(); - tensor_child->Resize(dim); + DDim dim = tensor_parent->dims(); + dim[0] = index_tensors[i].dims()[0]; tensor_child->mutable_data(dim, platform::CPUPlace()); Gather(dev_ctx.GetPlace(), tensor_parent, &index_tensors[i], @@ -174,32 +131,79 @@ void CondOp::Run(const Scope& scope, } } - // Step 3: run - for (int i = 0; i < 2; ++i) { - sub_net_op_[i]->Run(*sub_scopes[i], dev_ctx); + // create output_tensors in subscope for sub_net + for (int i = 0; i < BRANCH_NUM; ++i) { + for (auto& output : (*sub_net_op_[i]).Outputs()) { + for (auto& var_name : output.second) { + sub_scopes[i]->NewVar(var_name); + } + } } +} - // Step 4: merge output results +void CondOp::MergeDataFromSubnet(const framework::Scope& scope, + const platform::DeviceContext& dev_ctx) const { + std::vector& sub_scopes = GetSubScopes(scope); + const std::vector& index_tensors = + GetIndexTensors(scope); + + // Infer the output dim, out_dim[0] = true_dim[0] + false_dim[0] PADDLE_ENFORCE(!Outputs("Outs").empty(), "Outputs(Outs) of CondOp can't be empty."); - for (int i = 0; i < 2; ++i) { - // i= 0/i for True and False branches respectively - for (auto& output : Outputs("Outs")) { - // find Tensor - Variable* v = scope.FindVar(output); - PADDLE_ENFORCE_NOT_NULL(v); - LoDTensor* tensor_parent = v->GetMutable(); - - v = sub_scopes[i]->FindVar(output); - PADDLE_ENFORCE_NOT_NULL(v); - LoDTensor* tensor_child = v->GetMutable(); + for (auto& output : Outputs("Outs")) { + const LoDTensor* tensor_t_out = + &sub_scopes[TRUE_BRANCH]->FindVar(output)->Get(); + PADDLE_ENFORCE_NOT_NULL(tensor_t_out, "True output should not be NULL"); + const LoDTensor* tensor_f_out = + &sub_scopes[FALSE_BRANCH]->FindVar(output)->Get(); + PADDLE_ENFORCE_NOT_NULL(tensor_f_out, "False output should not be NULL"); + + auto* var_out = scope.FindVar(output); + PADDLE_ENFORCE_NOT_NULL(var_out, "Output not found"); + LoDTensor* tensor_out = var_out->GetMutable(); + PADDLE_ENFORCE_NOT_NULL(tensor_t_out, + "True output tensor should not be NULL"); + + DDim true_dim = tensor_t_out->dims(); + DDim false_dim = tensor_f_out->dims(); + true_dim[0] = 0; + false_dim[0] = 0; + PADDLE_ENFORCE_EQ(true_dim, false_dim, + "Outputs not of the same shape except the first dim"); + + DDim out_dim = tensor_t_out->dims(); + out_dim[0] = tensor_t_out->dims()[0] + tensor_f_out->dims()[0]; + tensor_out->Resize(out_dim); + tensor_out->mutable_data(platform::CPUPlace()); + } + // merge output results: + // output_tensor = true_output_tensor + false_output_tensor + for (auto& output : Outputs("Outs")) { + Variable* var_parent = scope.FindVar(output); + PADDLE_ENFORCE_NOT_NULL(var_parent); + auto* tensor_parent = var_parent->GetMutable(); + + for (int i = 0; i < BRANCH_NUM; ++i) { + Variable* var_child = sub_scopes[i]->FindVar(output); + PADDLE_ENFORCE_NOT_NULL(var_child); + auto* tensor_child = &var_child->Get(); ScatterUpdate(dev_ctx.GetPlace(), tensor_child, &index_tensors[i], tensor_parent); } } } +void CondOp::Run(const Scope& scope, + const platform::DeviceContext& dev_ctx) const { + PrepareDataForSubnet(scope, dev_ctx); + std::vector& sub_scopes = GetSubScopes(scope); + for (int i = 0; i < BRANCH_NUM; ++i) { + sub_net_op_[i]->Run(*sub_scopes[i], dev_ctx); + } + MergeDataFromSubnet(scope, dev_ctx); +} + class CondOpProtoAndCheckerMaker : public framework::OpProtoAndCheckerMaker { public: CondOpProtoAndCheckerMaker(framework::OpProto* proto, diff --git a/paddle/operators/cond_op.h b/paddle/operators/cond_op.h index 9a88ee35f1..93121fb31b 100644 --- a/paddle/operators/cond_op.h +++ b/paddle/operators/cond_op.h @@ -40,8 +40,7 @@ class CondOp : public framework::OperatorBase { const framework::VariableNameMap& outputs, const framework::AttributeMap& attrs) : OperatorBase(type, inputs, outputs, attrs) { - index_.resize(2); - sub_net_op_.resize(2); + sub_net_op_.resize(BRANCH_NUM); } CondOp(const CondOp& o) @@ -51,42 +50,44 @@ class CondOp : public framework::OperatorBase { PADDLE_THROW("Not implemented"); } - void CreateScope(const framework::Scope& scope) const; + framework::Scope& AddSubScope(const framework::Scope& scope) const; + std::vector& GetSubScopes( + const framework::Scope& scope) const; - void CreateIndexTensor(const framework::Scope& scope) const; + framework::LoDTensor& AddIndexTensor(const framework::Scope& scope) const; + std::vector& GetIndexTensors( + const framework::Scope& scope) const; - /* - * InferShape must be called before Run. - * FIXME(yuyang18): Since InferShape has been removed, this implementation - * could be wrong. - */ - void InferShape(const framework::Scope& scope) const; + void PrepareDataForSubnet(const framework::Scope& scope, + const platform::DeviceContext& dev_ctx) const; + void MergeDataFromSubnet(const framework::Scope& scope, + const platform::DeviceContext& dev_ctx) const; /* * Set True Block */ void set_truenet(std::unique_ptr&& net) { - sub_net_op_[0] = std::move(net); + sub_net_op_[TRUE_BRANCH] = std::move(net); } /* * Set False Block */ void set_falsenet(std::unique_ptr&& net) { - sub_net_op_[1] = std::move(net); + sub_net_op_[FALSE_BRANCH] = std::move(net); } void Run(const framework::Scope& scope, const platform::DeviceContext& dev_ctx) const override; private: + const int TRUE_BRANCH = 0; + const int FALSE_BRANCH = 1; + const int BRANCH_NUM = 2; + // sub_net_op_[0]: subnet_t // sub_net_op_[1]: subnet_f std::vector> sub_net_op_; - - // index_[0]: True_index; - // index_[1]: False_index; - mutable std::vector> index_; }; } // namespace operators diff --git a/paddle/operators/scatter.h b/paddle/operators/scatter.h index 6b542675c2..fec1046a97 100644 --- a/paddle/operators/scatter.h +++ b/paddle/operators/scatter.h @@ -78,10 +78,6 @@ void ScatterUpdate(const platform::Place& place, for (int i = 1; i < src_dims.size(); i++) PADDLE_ENFORCE(src_dims[i] == dst_dims[i]); - // slice size - size_t slice_size = 1; - for (int i = 0; i < src_dims.size(); ++i) slice_size *= src_dims[i]; - if (platform::is_cpu_place(place)) { CPUScatterUpdate(src, index->data(), index_size, output); } else { diff --git a/python/paddle/v2/framework/tests/test_cond_op.py b/python/paddle/v2/framework/tests/test_cond_op.py index e7a506f277..3698ce9c8e 100644 --- a/python/paddle/v2/framework/tests/test_cond_op.py +++ b/python/paddle/v2/framework/tests/test_cond_op.py @@ -112,7 +112,4 @@ class TestCondOp(unittest.TestCase): if __name__ == "__main__": - exit( - 0 - ) # FIXME(yuyang18): Since infer_shape has been removed, cond op may error unittest.main() From 2d8a5b97cc9096a84483b92be458d1388533c84f Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Fri, 29 Sep 2017 20:33:38 +0800 Subject: [PATCH 054/170] fix unit test --- paddle/operators/math/CMakeLists.txt | 5 ++- .../v2/framework/tests/test_pool2d_op.py | 36 ++++++++++++------- .../v2/framework/tests/test_pool3d_op.py | 32 +++++++++++------ 3 files changed, 48 insertions(+), 25 deletions(-) diff --git a/paddle/operators/math/CMakeLists.txt b/paddle/operators/math/CMakeLists.txt index f3ed0c78be..a0ceb029e3 100644 --- a/paddle/operators/math/CMakeLists.txt +++ b/paddle/operators/math/CMakeLists.txt @@ -1,11 +1,10 @@ if(WITH_GPU) - nv_library(math_function SRCS math_function.cc math_function.cu im2col.cc im2col.cu pooling.cc pooling.cu DEPS cblas device_context) + nv_library(math_function SRCS math_function.cc math_function.cu im2col.cc im2col.cu pooling.cc pooling.cu DEPS cblas device_context operator) nv_test(math_function_test SRCS math_function_test.cc DEPS math_function tensor) nv_library(softmax SRCS softmax.cc softmax.cu DEPS operator) nv_library(cross_entropy SRCS cross_entropy.cc cross_entropy.cu DEPS operator) else() - cc_library(math_function SRCS math_function.cc im2col.cc pooling.cc - DEPS cblas device_context operator) + cc_library(math_function SRCS math_function.cc im2col.cc pooling.cc DEPS cblas device_context operator) cc_test(math_function_test SRCS math_function_test.cc DEPS math_function tensor) cc_library(softmax SRCS softmax.cc DEPS operator) cc_library(cross_entropy SRCS cross_entropy.cc DEPS operator) diff --git a/python/paddle/v2/framework/tests/test_pool2d_op.py b/python/paddle/v2/framework/tests/test_pool2d_op.py index e8e5218073..0f9a548f29 100644 --- a/python/paddle/v2/framework/tests/test_pool2d_op.py +++ b/python/paddle/v2/framework/tests/test_pool2d_op.py @@ -47,7 +47,6 @@ def avg_pool2D_forward_naive(x, ksize, strides, paddings=[0, 0], global_pool=0): class TestPool2d_Op(OpTest): def setUp(self): self.initTestCase() - self.op_type = "pool2d" input = np.random.random(self.shape).astype("float32") output = self.pool2D_forward_naive(input, self.ksize, self.strides, self.paddings, self.global_pool) @@ -71,7 +70,8 @@ class TestPool2d_Op(OpTest): self.check_grad(set(['X']), 'Out', max_relative_error=0.07) def initTestCase(self): - self.global_pool = 0 + self.global_pool = True + self.op_type = "pool2d" self.pool_type = "avg" self.pool2D_forward_naive = avg_pool2D_forward_naive self.shape = [2, 3, 5, 5] @@ -82,23 +82,23 @@ class TestPool2d_Op(OpTest): class TestCase1(TestPool2d_Op): def initTestCase(self): - self.global_pool = 0 + self.global_pool = False self.op_type = "pool2d" self.pool_type = "avg" self.pool2D_forward_naive = avg_pool2D_forward_naive - self.shape = [2, 3, 5, 5] + self.shape = [2, 3, 7, 7] self.ksize = [3, 3] self.strides = [1, 1] - self.paddings = [1, 1] + self.paddings = [0, 0] class TestCase2(TestPool2d_Op): def initTestCase(self): - self.global_pool = 1 + self.global_pool = False self.op_type = "pool2d" self.pool_type = "avg" self.pool2D_forward_naive = avg_pool2D_forward_naive - self.shape = [2, 3, 5, 5] + self.shape = [2, 3, 7, 7] self.ksize = [3, 3] self.strides = [1, 1] self.paddings = [1, 1] @@ -106,23 +106,35 @@ class TestCase2(TestPool2d_Op): class TestCase3(TestPool2d_Op): def initTestCase(self): - self.global_pool = 0 + self.global_pool = True self.op_type = "pool2d" self.pool_type = "max" self.pool2D_forward_naive = max_pool2D_forward_naive self.shape = [2, 3, 5, 5] self.ksize = [3, 3] self.strides = [1, 1] - self.paddings = [1, 1] + self.paddings = [0, 0] -class TestCase4(TestPool2d_Op): +class TestCase3(TestPool2d_Op): def initTestCase(self): - self.global_pool = 1 + self.global_pool = False self.op_type = "pool2d" self.pool_type = "max" self.pool2D_forward_naive = max_pool2D_forward_naive - self.shape = [2, 3, 5, 5] + self.shape = [2, 3, 7, 7] + self.ksize = [3, 3] + self.strides = [1, 1] + self.paddings = [0, 0] + + +class TestCase3(TestPool2d_Op): + def initTestCase(self): + self.global_pool = False + self.op_type = "pool2d" + self.pool_type = "max" + self.pool2D_forward_naive = max_pool2D_forward_naive + self.shape = [2, 3, 7, 7] self.ksize = [3, 3] self.strides = [1, 1] self.paddings = [1, 1] diff --git a/python/paddle/v2/framework/tests/test_pool3d_op.py b/python/paddle/v2/framework/tests/test_pool3d_op.py index 2a3de89a70..f041ff9178 100644 --- a/python/paddle/v2/framework/tests/test_pool3d_op.py +++ b/python/paddle/v2/framework/tests/test_pool3d_op.py @@ -55,7 +55,6 @@ def avg_pool3D_forward_naive(x, ksize, strides, paddings=[0, 0], global_pool=0): class TestPool3d_Op(OpTest): def setUp(self): self.initTestCase() - self.op_type = "pool3d" input = np.random.random(self.shape).astype("float32") output = self.pool3D_forward_naive(input, self.ksize, self.strides, self.paddings, self.global_pool) @@ -79,7 +78,8 @@ class TestPool3d_Op(OpTest): self.check_grad(set(['X']), 'Out', max_relative_error=0.07) def initTestCase(self): - self.global_pool = 0 + self.global_pool = True + self.op_type = "pool3d" self.pool_type = "avg" self.pool3D_forward_naive = avg_pool3D_forward_naive self.shape = [2, 3, 5, 5, 5] @@ -90,19 +90,19 @@ class TestPool3d_Op(OpTest): class TestCase1(TestPool3d_Op): def initTestCase(self): - self.global_pool = 0 + self.global_pool = False self.op_type = "pool3d" self.pool_type = "avg" self.pool3D_forward_naive = avg_pool3D_forward_naive self.shape = [2, 3, 7, 7, 7] self.ksize = [3, 3, 3] self.strides = [1, 1, 1] - self.paddings = [1, 1, 1] + self.paddings = [0, 0, 0] class TestCase2(TestPool3d_Op): def initTestCase(self): - self.global_pool = 1 + self.global_pool = False self.op_type = "pool3d" self.pool_type = "avg" self.pool3D_forward_naive = avg_pool3D_forward_naive @@ -114,23 +114,35 @@ class TestCase2(TestPool3d_Op): class TestCase3(TestPool3d_Op): def initTestCase(self): - self.global_pool = 0 + self.global_pool = True self.op_type = "pool3d" self.pool_type = "max" self.pool3D_forward_naive = max_pool3D_forward_naive self.shape = [2, 3, 5, 5, 5] self.ksize = [3, 3, 3] self.strides = [1, 1, 1] - self.paddings = [1, 1, 1] + self.paddings = [0, 0, 0] -class TestCase4(TestPool3d_Op): +class TestCase3(TestPool3d_Op): def initTestCase(self): - self.global_pool = 1 + self.global_pool = False self.op_type = "pool3d" self.pool_type = "max" self.pool3D_forward_naive = max_pool3D_forward_naive - self.shape = [2, 3, 5, 5, 5] + self.shape = [2, 3, 7, 7, 7] + self.ksize = [3, 3, 3] + self.strides = [1, 1, 1] + self.paddings = [0, 0, 0] + + +class TestCase3(TestPool3d_Op): + def initTestCase(self): + self.global_pool = False + self.op_type = "pool3d" + self.pool_type = "max" + self.pool3D_forward_naive = max_pool3D_forward_naive + self.shape = [2, 3, 7, 7, 7] self.ksize = [3, 3, 3] self.strides = [1, 1, 1] self.paddings = [1, 1, 1] From 6abcb74c8f8918a52c7016cb4cc924635f6f3539 Mon Sep 17 00:00:00 2001 From: chengduoZH Date: Sat, 30 Sep 2017 10:49:02 +0800 Subject: [PATCH 055/170] fix unit test class name --- python/paddle/v2/framework/tests/test_pool2d_op.py | 4 ++-- python/paddle/v2/framework/tests/test_pool3d_op.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/python/paddle/v2/framework/tests/test_pool2d_op.py b/python/paddle/v2/framework/tests/test_pool2d_op.py index 0f9a548f29..2941fda81b 100644 --- a/python/paddle/v2/framework/tests/test_pool2d_op.py +++ b/python/paddle/v2/framework/tests/test_pool2d_op.py @@ -116,7 +116,7 @@ class TestCase3(TestPool2d_Op): self.paddings = [0, 0] -class TestCase3(TestPool2d_Op): +class TestCase4(TestPool2d_Op): def initTestCase(self): self.global_pool = False self.op_type = "pool2d" @@ -128,7 +128,7 @@ class TestCase3(TestPool2d_Op): self.paddings = [0, 0] -class TestCase3(TestPool2d_Op): +class TestCase5(TestPool2d_Op): def initTestCase(self): self.global_pool = False self.op_type = "pool2d" diff --git a/python/paddle/v2/framework/tests/test_pool3d_op.py b/python/paddle/v2/framework/tests/test_pool3d_op.py index f041ff9178..8792b492e3 100644 --- a/python/paddle/v2/framework/tests/test_pool3d_op.py +++ b/python/paddle/v2/framework/tests/test_pool3d_op.py @@ -124,7 +124,7 @@ class TestCase3(TestPool3d_Op): self.paddings = [0, 0, 0] -class TestCase3(TestPool3d_Op): +class TestCase4(TestPool3d_Op): def initTestCase(self): self.global_pool = False self.op_type = "pool3d" @@ -136,7 +136,7 @@ class TestCase3(TestPool3d_Op): self.paddings = [0, 0, 0] -class TestCase3(TestPool3d_Op): +class TestCase5(TestPool3d_Op): def initTestCase(self): self.global_pool = False self.op_type = "pool3d" From e6f2da489c2d69bae2150edd7537d6d40094b8b3 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Fri, 29 Sep 2017 20:19:55 -0700 Subject: [PATCH 056/170] Update --- doc/design/python_api.md | 220 ++++++++++++++++++++++----------------- 1 file changed, 125 insertions(+), 95 deletions(-) diff --git a/doc/design/python_api.md b/doc/design/python_api.md index 3122f5bfd8..5c68354274 100644 --- a/doc/design/python_api.md +++ b/doc/design/python_api.md @@ -1,174 +1,206 @@ # Design Doc: Python API -The top level user API in Python should be as same as API in `paddle.v2` after refactoring Paddle from a layer based framework to an operator based framework. There are many new classes in C++ in [compile time] for describing neural networks, such as `Variable`, `Operator`, `Block`. The issue about current design is how to give a proper way to wrap the C++ API to `paddle.v2` API and write layers in Python. +Due to the refactorization of the PaddlePaddle core, we need Python classes to construct corresponding protobuf messages that describe a DL program. -This implementation of Python API includes two steps. - -1. Implement the Python API using current C++ runtime concepts. -2. Replace the implementation by using compile-time concepts when they are completed. - -The implementation of the first step is a temporary implementation. We should design our Python API concepts based on `compile-time` concepts. We just use `runtime` classes to implement it for now. - - -## Python Class and compile-time protobuf - -Since we design our Python API concepts based on `compile-time`, we try to map our Python classes to every compile-time result, i.e., the protobuf messages. They are: - - -| Python Class | Compile-time protobuf | +| Python classes | Protobuf messages | | --- | --- | | Program | ProgramDesc | | Block | BlockDesc | | Operator | OpDesc | | Variable | VarDesc | +Please be aware that these Python classes need to maintain some construction-time information, which are not part of the protobuf messages. + +## Core Concepts + ### Program -`Program` is the description of the whole training process and there can only be one `Program` object, which is created automatically by the system at the very beginning. `Program` is formed by a series of `Block`. +A `ProgramDesc` describes a [DL program](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/program.md), which is composed of an array of `BlockDesc`s. A `BlockDesc` refers to its parent block by its index in the array. For example, operators in the step block of an RNN operator needs to be able to access variables in its ancessor blocks. + +Whenever we create a block, we need set its parent block to the current block, so the Python class `Program` needs to maintain a data member `current_block`. ```python class Program(objects): def __init__(self): + self.proto = core.NewProgram() # a C++ ProgramDesc pointer. self.blocks = vector() - self.blocks.append(Block(None)) - self.current_block_idx = 0 + self.blocks.append(Block(self, -1)) # the global block + self.current_block = 0 # initialized to the global block - def get_block(block_idx): - return self.blocks[block_idx] + def global_block(): + return self.blocks[0] def current_block(): - return self.get_block(self.current_block_idx) - - def fallback_current_block(): - self.current_block_idx = self.current_block().parent_idx + return self.get_block(self.current_block) + def rollback(): + self.current_block = self.current_block().parent_idx def create_block(): new_block_idx = len(self.block) - self.blocks.append(Block(parent_idx=self.current_block_idx, - idx=new_block_idx)) - self.current_block_idx = new_block_idx + self.blocks.append(Block(self, self.current_block)) + self.current_block = new_block_idx + return current_block() ``` -`Program` will create the first block in its constructor. The first block is called 'global block'. It is where all parameters are stored. +`Program` is an accessor to the protobuf message `ProgramDesc`, which is created in C++ space, because the InferShape function is in C++, which manipulates `VarDesc` messages, which are in turn members of `BlockDesc`, which is a member of `ProgramDesc`. -### Block +`Program` creates the first block as the global block in its constructor. All parameters and their initializer operators are in the global block. -Block is just like programming languages `{}`, which contains many operators and variables. There are two data fields in `Block`. 1) An associate map, whose key is variable name and value is variable itself; 2) A list of operators. +### Block -The block is hierarchical because PaddlePaddle supports RNN and IfElse. For example, RNN is like `for-loop` in programming languages. There is new `block` inside a `for-loop`. To represent hierarchies, `Block` stores the index of `parent Block` inside. The 'index' means the block's position in `Program`'s `blocks`. If `parent_idx=None`, the block itself is the outermost block, i.e., the 'global block'. +A [Block](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/block.md) includes +1. a map from variable names to an instance of the Python `Variable` class, and +1. a list of `Operator` instances. ```python class Block(objects): - def __init__(self, parent_idx, idx): + def __init__(self, program, parent_idx): + self.proto = core.NewBlock(program.proto) + self.program = program self.vars = map() self.ops = vector() - self.idx = idx self.parent_idx = parent_idx - + def create_var(self, ...): - # create variable in `self.vars` - return Variable(...) - - - def create_global_var(self, ...): - if self.parent_idx is not None: - parent_block = program.get_block(parent_idx) - return parent_block.create_global_var(...) - else: - return self.create_var(...) - - def create_parameter(self, ...): - return self.create_global_var(...) - + return Variable(self, ...) + + def _create_global_var(self, ...): + program.global_block().create_var(...) + + def create_parameter(self, name, ...): + # Parameter is a subclass of variable. See Parameter section for details. + self.vars[name] = Parameter(self._create_global_var(...), ...) + return self.vars[name] + def append_operator(self, ...): - self.ops.append(...) - - def prepend_operator(self, ...): - self.ops.prepend(...) -``` + self.ops.append(Operator(self, ...)) -Users are able to create a global variable inside any block since they many create parameters inside a RNN or IfElse. All parameters should be stored in the global block, not the step block in RNN. + def prepend_operator(self, ...): # Parameter's ctor prepands initialize operators. + self.ops.prepend(Operator(self, ...)) +``` -Users can create local variables for outputs of operators. Users can also append and prepend an operator in current block. Prepending `random initialize` operator or `load` operator is very useful to initialize parameters before training. +`create_parameter` is necessary because parameters are global variables, those defined in the global block, but can be created in some sub-blocks, e.g., an FC layer in the step block of an RNN operator. +`prepand_operator` is necessary because the constructor of `Parameter` needs to create the initialize (or load) operator of the parameter, and would like to put it in the *preamble* of the global block. ### Operator -Operator class will take inputs, outputs and attributes of the operator into `protobuf` OpDesc and create a C++ `OpDesc` instance. The `infer_shape` perform on C++ objects. +The `Operator` class fills in the `OpDesc` message and calls the C++ function `InferShape` to infer output shape from input shape. ```python class Operator(object): - def __init__(self, type, inputs, outputs, attrs): - # create OpDesc in Python - op_desc = ... - self.cpp_op_desc_ptr = core.OpDesc(op_desc) - cpp.infer_shape(self.cpp_op_desc_ptr, inputs, outputs) + def __init__(self, + block, # Block + type, # string + inputs, # dict + outputs,# dict + attrs # dict + ): + self.proto = core.NewOpDesc(block.proto, type, inputs, outputs, attrs) + core.infer_shape(self.proto, inputs, outputs) def type(self): - return self.cpp_op_desc_ptr.type() + return self.proto.type() ``` -After creating a C++ `OpDesc`, `Operator` in Python can only reads the attribute from C++ side. +`Operator` creates the `OpDesc` message in C++ space, so could it call the `InferShape` function, which is in C++. ### Variable -Operators' inputs, outputs, and parameters are all variables. In our design, a variable has four key attributes: its name(`name`), the block it belongs to(`block`), a pointer pointed to its C++ Protobuf object(`cpp_var_desc_ptr`), and the operator it is created by(`op`). All of these attributes are initialized in the constructor, except the `op`. The `op` will keep being `None` till the variable is taken as an operator's output. +Operators take Variables as its inputs and outputs. ```python class Variable(object): - def __init__(self, shape, dtype="float32", name=None, block=None): + def __init__(self, + block=None, # Block + name=None, # string + shape, # tuple + dtype="float32", # string + lod_level=None # int + ): if name is None: name = unique_name_generator() self.name = name self.block = block - # build C++ Protobuf object - self.cpp_var_desc_ptr = ... - self.op = None - - def shape(self): - cpp_shape = self.cpp_var_desc_ptr.shape() - return [None if elem < 0 else elem for elem in cpp_shape] + self.proto = core.NewVarDesc(block.proto, name, shape, lod_level) + self.writer = None ``` -The Protobuf object should be created in C++ not Python because it is needed by infershape, and infershape is implemented by C++ code. The C++ Protobuf object is accessible for Python through the `cpp_var_desc_ptr`, just like how `shape()` function does. - -The user is allowed to build a variable without specifying its name. If so, it is going to be assigned with an automatically generated unique name. +Please be aware of `self.writer`, that tracks operator who creates the variable. It possible that there are more than one operators who write a variable, but in Python space, each writes to a variable is represented by a Variable class. This is guaranteed by the fact that **`core.NewVarDesc` must NOT create a new `VarDesc` message if its name already exists in the specified block**. ### Parameter -The parameter is a kind of special variable. They need to be initialized at the very beginning and updated after each batch training. So if a variable is a parameter, our compiler will add an initializer op and an optimizer op for it during the building process of computation graph. Apart from these, there is no more difference between variable and parameter. In other words, 'parameter' is only a label attached to variables, to tell the compiler these ones require additional processing. +A parameter is a global variable with an initializer (or load) operator. ```python class Parameter(Variable): - def __init__(self, trainable, initialize_attrs, optimize_attrs): - pass + def __init__(self, + block=None, # Block + name=None, # string + shape, # tuple + dtype="float32", # string + lod_level=None # int + trainable, # bool + initialize_op_attrs, + optimize_op_attrs): + super(Parameter, self).__init__(block, name, shape, dtype, lod_level) + self.trainable = trainable + self.optimize_op_attrs = optimize_op_attrs + block.prepend(Operator(block, # Block + initialize_op_attrs['type'], # string + None, # no inputs + self, # output is the parameter + initialize_op_attrs) ``` -The class `Parameter` is derived from class `Variable`. In addition to variables have, parameters are able to hold their initializing and updating information. A parameter's `self.op` will always be `None` because it can never be an operator's output. +When users create a parameter, s/he can call +```python +program.create_parameter( + ..., + init_attr={ + type: "uniform_random", + min: -1.0, + max: 1.0, + }) +) +``` -## Layer Functions +In above example, `init_attr.type` names an initialize operator. It can also name the load operator + +```python +init_attr={ + type: "load", + filename: "something.numpy", +} +``` -A layer is a Python function. When it is invoked, it creates a series of operators and variables then inserts them into the block. It is something like the macro in C++. It is called 'Layer' because the combination of added operators acts just like what a neural network layer does. +`optimize_op_attrs` is not in the `VarDesc` message, but kept in the Python instance, as it will be used in the Python space when creating the optimize operator's `OpDesc`, and will be in the `OpDesc` message. + +## Layer Functions -Here are examples of how to write a data layer and FC layer: +A layer is a Python function that creates some operators and variables. Layers simplify the work of application programmers. ### Data Layer ```python -def data_layer(name, type): - block = program.current_block() - # type = dense_vector(size=10) / integer_value(range=10) - return block.create_global_var( - name=name, - shape=[None] + type.dims(), +def data_layer(name, type, column_name): + block = the_current_program.glolal_block() + var = block.create_global_var( + name=name, + shape=[None] + type.dims(), dtype=type.dtype) + block.prepend_operator(block, + type="Feed", + inputs = None, + outputs = [var], + {column_name: column_name}) + return var +``` -``` - -All the new variables and operators will be built in the `current block`. In the above `data_layer` code, a variable is created and be inserted into the root block to make it global. This variable is going to be used as input data of the whole network. +The input to the feed operator is a special variable in the global scope, which is the output of [Python readers](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/reader/README.md). ### FC Layer @@ -178,9 +210,7 @@ def fc_layer(input, size, ...): w = block.create_parameter(...) b = block.create_parameter(...) out = block.create_var() - op = block.append_operator(Operator("FC", X=input, W=w, b=b, Out=out)) - out.op = op + op = block.append_operator("FC", X=input, W=w, b=b, out=out) + out.writer = op return out ``` - -In the `fc_layer` code, we create two parameters(`w` and `b`), one variable(`out`) and one operator(`FC operator`), then insert all of them into the `current block`. From 2d8c7e310a23f62f3a9aad7823beec8f714bd368 Mon Sep 17 00:00:00 2001 From: Yancey Date: Sat, 30 Sep 2017 12:10:21 +0800 Subject: [PATCH 057/170] Add Inputs/Outputs check interface (#4438) Add multiple Inputs/Outputs check interface --- paddle/framework/operator.h | 26 ++++++++++++++++++++++++++ paddle/framework/shape_inference.h | 4 ++++ 2 files changed, 30 insertions(+) diff --git a/paddle/framework/operator.h b/paddle/framework/operator.h index 310d68d7c1..0af527c88c 100644 --- a/paddle/framework/operator.h +++ b/paddle/framework/operator.h @@ -334,6 +334,32 @@ class RuntimeInferShapeContext : public InferShapeContextBase { return var != nullptr; } + bool HasInputs(const std::string& name) const { + auto inputs = op_.Inputs(name); + if (inputs.size() == 0UL) { + return false; + } + for (auto& input : inputs) { + if (scope_.FindVar(input) == nullptr) { + return false; + } + } + return true; + } + + bool HasOutputs(const std::string& name) const { + auto outputs = op_.Outputs(name); + if (outputs.size() == 0UL) { + return false; + } + for (auto& output : outputs) { + if (scope_.FindVar(output) == nullptr) { + return false; + } + } + return true; + } + DDim GetInputDim(const std::string& name) const { return GetDim(op_.Input(name)); } diff --git a/paddle/framework/shape_inference.h b/paddle/framework/shape_inference.h index b07fc78812..bc8af0eb3e 100644 --- a/paddle/framework/shape_inference.h +++ b/paddle/framework/shape_inference.h @@ -24,6 +24,10 @@ class InferShapeContextBase { virtual ~InferShapeContextBase() {} virtual bool HasInput(const std::string &name) const = 0; virtual bool HasOutput(const std::string &name) const = 0; + + virtual bool HasInputs(const std::string &name) const = 0; + virtual bool HasOutputs(const std::string &name) const = 0; + virtual framework::DDim GetInputDim(const std::string &name) const = 0; std::vector GetInputsDim(const std::string &name) const { const std::vector &names = Inputs(name); From 8676423bc9a32496f2aca57e3853f743786fce21 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Fri, 29 Sep 2017 23:07:08 -0700 Subject: [PATCH 058/170] Update --- doc/design/register_grad_op.md | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/doc/design/register_grad_op.md b/doc/design/register_grad_op.md index 02c4d554d0..12b04fb271 100644 --- a/doc/design/register_grad_op.md +++ b/doc/design/register_grad_op.md @@ -1,8 +1,17 @@ -# Design Doc: Register Gradient Operator +# Design Doc: Gradient Operators Registration -## Problem -Since we separate users program in two stages, compile time and runtime, we should record and look up the mapping relationship between an operator and its gradient operators when compile. However, we register this relationship in runtime by these `OpInfo` fields. +## The Problem Posed + +In our current operator registration mechanism, for each operator, the programmer should register a *gradient operator creator* function, which takes a C++ operator instance, and returns the corresponding gradient instance. + +However, as we decided to separate the *compilation* and *execution* of DL models, we need to reshape the creator to take a protobuf `OpDesc` message, and returns a corresponding message. + +More than that, the new registration mechanism need to support the fact that an operators' gradient computation might be a composition of operators. + +## Current Implementation + +OpInfos store in a association map which key is the operator type. The `grad_op_type` indicate associated gradient operator type. Operator can create gradient operator by `OpInfo::creator_` of gradient. The pseudo code is ```cpp struct OpInfo { @@ -10,11 +19,7 @@ struct OpInfo { std::string grad_op_type_; ... }; -``` -OpInfos store in a association map which key is the operator type. The `grad_op_type` indicate associated gradient operator type. Operator can create gradient operator by `OpInfo::creator_` of gradient. The pseudo code is - -```cpp map OpInfoMap; OperatorBase* CreateGradientOperator(const OperatorBase& op) { @@ -22,14 +27,7 @@ OperatorBase* CreateGradientOperator(const OperatorBase& op) { } ``` -At the same time, an operator's gradient operator could be composed of many forward operators. For example, the gradient operator of `minus_op` could consist of an `identity` operator and a `scale` operator. To compose a gradient operator by forwarding operators could: 1) Reuse forwarding operator; 2) Calculate second derivative, third derivative, etc. - -We use `NetOp` to represent a composed operator since the `NetOp` is `vector`. However, `NetOp` is also a runtime concept. We should provide a mechanism to compose operators as a gradient operator. - -In conclusion, the problem that we want to resolve in this design doc is to register the mapping relationship between the forward operator and its gradient operators during compile time. - - -## Solution +## Proposed Solution The mapping relationship between an operator and its gradient operators is a function. The interface of that function is: From 480154896cf3d32b646f3cab5b546c37fb2866aa Mon Sep 17 00:00:00 2001 From: caoying03 Date: Sat, 30 Sep 2017 16:06:52 +0800 Subject: [PATCH 059/170] add configuration helper for resize layer. --- doc/api/v1/index_cn.rst | 2 +- doc/api/v2/config/layer.rst | 5 ++++ .../paddle/trainer_config_helpers/layers.py | 25 ++++++++++++++++- .../tests/configs/file_list.sh | 2 +- .../protostr/test_resize_layer.protostr | 27 +++++++++++++++++++ .../tests/configs/test_resize_layer.py | 6 +++++ 6 files changed, 64 insertions(+), 3 deletions(-) create mode 100644 python/paddle/trainer_config_helpers/tests/configs/protostr/test_resize_layer.protostr create mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_resize_layer.py diff --git a/doc/api/v1/index_cn.rst b/doc/api/v1/index_cn.rst index 3718cd73a2..cf146dc088 100644 --- a/doc/api/v1/index_cn.rst +++ b/doc/api/v1/index_cn.rst @@ -21,7 +21,7 @@ Model Config API trainer_config_helpers/optimizers.rst trainer_config_helpers/data_sources.rst trainer_config_helpers/layers.rst - trainer_config_helpers/activations.rst + trainer_config_helpers/activations.rst trainer_config_helpers/poolings.rst trainer_config_helpers/networks.rst trainer_config_helpers/evaluators.rst diff --git a/doc/api/v2/config/layer.rst b/doc/api/v2/config/layer.rst index c94627a728..d4e9d53e5c 100644 --- a/doc/api/v2/config/layer.rst +++ b/doc/api/v2/config/layer.rst @@ -345,6 +345,11 @@ clip .. autoclass:: paddle.v2.layer.clip :noindex: +resize +------ +.. autoclass:: paddle.v2.layer.resize + :noindex: + slope_intercept --------------- .. autoclass:: paddle.v2.layer.slope_intercept diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index 74025d2a7b..d37f29d2c4 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -142,6 +142,7 @@ __all__ = [ 'img_pool3d_layer', 'scale_shift_layer', 'img_conv3d_layer', + 'resize_layer', ] @@ -250,6 +251,8 @@ class LayerType(object): KMAX_SEQ_SCORE = 'kmax_seq_score' SCALE_SHIFT_LAYER = 'scale_shift' + RESIZE = 'resize' + @staticmethod def is_layer_type(type_name): """ @@ -6473,7 +6476,7 @@ def switch_order_layer(input, act=None, layer_attr=None): """ - This layer switch dimension order of image input. + This layer switch dimension order of image input. From order "batchSize, channels, height, width" to order "batchSize, height, width, channels". @@ -6932,3 +6935,23 @@ def scale_shift_layer(input, name=None, param_attr=None, bias_attr=None): bias=ParamAttr.to_bias(bias_attr)) return LayerOutput( name, LayerType.SCALE_SHIFT_LAYER, parents=[input], size=input.size) + + +@wrap_name_default("resize") +def resize_layer(input, size, name=None): + """ + The resize layer resizes the input matrix with a shape of [Height, Width] + into the output matrix with a shape of [Height x Width / size, size], + where size is the parameter of this layer indicating the output dimension. + + :param input: The input to this layer. + :type input: LayerOutput. + :param name: The name of this layer. It is optional. + :type name: basestring + :param size: The resized output dimesion of this layer. + :type size: int + :return: A LayerOutput object. + :rtype: LayerOutput + """ + Layer(name=name, type=LayerType.RESIZE, inputs=Input(input.name), size=size) + return LayerOutput(name, LayerType.RESIZE, parents=[input], size=input.size) diff --git a/python/paddle/trainer_config_helpers/tests/configs/file_list.sh b/python/paddle/trainer_config_helpers/tests/configs/file_list.sh index 8a204a96f3..6a4550c209 100755 --- a/python/paddle/trainer_config_helpers/tests/configs/file_list.sh +++ b/python/paddle/trainer_config_helpers/tests/configs/file_list.sh @@ -10,6 +10,6 @@ test_prelu_layer test_row_conv test_detection_output_layer test_multibox_loss_la test_recursive_topology test_gated_unit_layer test_clip_layer test_row_l2_norm_layer test_kmax_seq_socre_layer test_sub_nested_seq_select_layer test_scale_shift_layer test_seq_slice_layer test_cross_entropy_over_beam test_pooling3D_layer -test_conv3d_layer test_deconv3d_layer test_BatchNorm3D) +test_conv3d_layer test_deconv3d_layer test_BatchNorm3D test_resize_layer) export whole_configs=(test_split_datasource) diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_resize_layer.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_resize_layer.protostr new file mode 100644 index 0000000000..9399252b23 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_resize_layer.protostr @@ -0,0 +1,27 @@ +type: "nn" +layers { + name: "input" + type: "data" + size: 300 + active_type: "" +} +layers { + name: "__resize_0__" + type: "resize" + size: 150 + active_type: "" + inputs { + input_layer_name: "input" + } +} +input_layer_names: "input" +output_layer_names: "__resize_0__" +sub_models { + name: "root" + layer_names: "input" + layer_names: "__resize_0__" + input_layer_names: "input" + output_layer_names: "__resize_0__" + is_recurrent_layer_group: false +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_resize_layer.py b/python/paddle/trainer_config_helpers/tests/configs/test_resize_layer.py new file mode 100644 index 0000000000..09a6f50733 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/test_resize_layer.py @@ -0,0 +1,6 @@ +from paddle.trainer_config_helpers import * + +data = data_layer(name='input', size=300) +resized = resize_layer(input=data, size=150) + +outputs(resized) From d9e3c4ff358cb6d9262375e37f3cc7fdd0dc7465 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Sat, 30 Sep 2017 12:51:05 -0700 Subject: [PATCH 060/170] Add GradOpDescMaker to OpInfo and complete OperatorRegistrar method --- paddle/framework/CMakeLists.txt | 2 +- paddle/framework/details/op_registry.h | 105 +++++++++++++++++++++++++ paddle/framework/op_info.h | 13 ++- paddle/framework/op_registry.h | 49 +++++------- paddle/framework/op_registry_test.cc | 11 +++ 5 files changed, 148 insertions(+), 32 deletions(-) create mode 100644 paddle/framework/details/op_registry.h diff --git a/paddle/framework/CMakeLists.txt b/paddle/framework/CMakeLists.txt index 8a5d8532bb..feadc9be4b 100644 --- a/paddle/framework/CMakeLists.txt +++ b/paddle/framework/CMakeLists.txt @@ -22,7 +22,7 @@ cc_library(attribute SRCS attribute.cc DEPS framework_proto) cc_library(proto_desc SRCS var_desc.cc op_desc.cc block_desc.cc program_desc.cc DEPS attribute) cc_library(op_proto_maker SRCS op_proto_maker.cc DEPS framework_proto attribute) cc_test(op_proto_maker_test SRCS op_proto_maker_test.cc DEPS op_proto_maker) -cc_library(op_info SRCS op_info.cc DEPS attribute framework_proto) +cc_library(op_info SRCS op_info.cc DEPS attribute framework_proto proto_desc) cc_library(operator SRCS operator.cc DEPS op_info device_context tensor scope) cc_test(operator_test SRCS operator_test.cc DEPS operator op_registry) diff --git a/paddle/framework/details/op_registry.h b/paddle/framework/details/op_registry.h new file mode 100644 index 0000000000..d2516ccc1e --- /dev/null +++ b/paddle/framework/details/op_registry.h @@ -0,0 +1,105 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once + +#include "paddle/framework/op_info.h" +#include "paddle/framework/op_proto_maker.h" +#include "paddle/framework/operator.h" + +namespace paddle { +namespace framework { +namespace details { + +enum OpInfoFillType { + kOperator = 0, + kOpProtoAndCheckerMaker = 1, + kGradOpDescMaker = 2 +}; + +template +struct OpInfoFillTypeID { + static constexpr OpInfoFillType ID() { + return std::is_base_of::value + ? kOperator + : (std::is_base_of::value + ? kOpProtoAndCheckerMaker + : (std::is_base_of::value + ? kGradOpDescMaker + : static_cast(-1))); + } +}; + +template ::ID()> +struct OpInfoFiller; + +template +class OperatorRegistrarRecursive; + +template +class OperatorRegistrarRecursive { + public: + using T = typename std::tuple_element>::type; + OperatorRegistrarRecursive(const char* op_type, OpInfo* info) { + OpInfoFiller fill; + fill(op_type, info); + constexpr auto size = sizeof...(ARGS); + OperatorRegistrarRecursive reg(op_type, + info); + (void)(reg); + } +}; + +template +class OperatorRegistrarRecursive { + public: + OperatorRegistrarRecursive(const char* op_type, OpInfo* info) {} +}; + +template +struct OpInfoFiller { + void operator()(const char* op_type, OpInfo* info) const { + info->creator_ = [](const std::string& type, const VariableNameMap& inputs, + const VariableNameMap& outputs, + const AttributeMap& attrs) { + return new T(type, inputs, outputs, attrs); + }; + } +}; + +template +struct OpInfoFiller { + void operator()(const char* op_type, OpInfo* info) const { + info->proto_ = new OpProto; + info->checker_ = new OpAttrChecker(); + auto maker = T(info->proto_, info->checker_); + maker.Validate(); + info->proto_->set_type(op_type); + PADDLE_ENFORCE( + info->proto_->IsInitialized(), + "Fail to initialize %s's OpProto, because %s is not initialized", + op_type, info->proto_->InitializationErrorString()); + } +}; + +template +struct OpInfoFiller { + void operator()(const char* op_type, OpInfo* info) const { + info->grad_op_maker_ = new T(); + } +}; +} // namespace details + +} // namespace framework +} // namespace paddle diff --git a/paddle/framework/op_info.h b/paddle/framework/op_info.h index b98d8f23a1..6d1ee4dece 100644 --- a/paddle/framework/op_info.h +++ b/paddle/framework/op_info.h @@ -17,8 +17,8 @@ #include #include #include - #include "paddle/framework/attribute.h" +#include "paddle/framework/op_desc.h" namespace paddle { namespace framework { @@ -29,11 +29,18 @@ using OpCreator = std::function; +class GradOpDescMakerBase { + public: + virtual ~GradOpDescMakerBase() = default; + virtual std::vector operator()(const OpDescBind&) const = 0; +}; + struct OpInfo { OpCreator creator_; std::string grad_op_type_; - OpProto* proto_; - OpAttrChecker* checker_; + GradOpDescMakerBase* grad_op_maker_{nullptr}; + OpProto* proto_{nullptr}; + OpAttrChecker* checker_{nullptr}; bool HasOpProtoAndChecker() const { return proto_ != nullptr && checker_ != nullptr; diff --git a/paddle/framework/op_registry.h b/paddle/framework/op_registry.h index 4db38badae..4ee2c7d275 100644 --- a/paddle/framework/op_registry.h +++ b/paddle/framework/op_registry.h @@ -21,49 +21,42 @@ limitations under the License. */ #include #include #include "paddle/framework/attribute.h" +#include "paddle/framework/details/op_registry.h" #include "paddle/framework/framework.pb.h" #include "paddle/framework/grad_op_builder.h" -#include "paddle/framework/op_info.h" -#include "paddle/framework/op_proto_maker.h" #include "paddle/framework/operator.h" #include "paddle/framework/scope.h" namespace paddle { namespace framework { +template +struct OperatorRegistrar { + explicit OperatorRegistrar(const char* op_type) : op_type(op_type) { + PADDLE_ENFORCE(!OpInfoMap::Instance().Has(op_type), + "'%s' is registered more than once.", op_type); + static_assert(sizeof...(ARGS) != 0, + "OperatorRegistrar should be invoked at least by OpClass"); + details::OperatorRegistrarRecursive<0, false, ARGS...>(op_type, &info); + } + + ~OperatorRegistrar() { OpInfoMap::Instance().Insert(op_type, info); } + + const char* op_type; + + OpInfo info; +}; + class OpRegistry { public: template static void RegisterOp(const std::string& op_type, const std::string& grad_op_type) { - PADDLE_ENFORCE(!OpInfoMap::Instance().Has(op_type), - "'%s' is registered more than once.", op_type); - OpInfo op_info; - op_info.creator_ = []( - const std::string& type, const VariableNameMap& inputs, - const VariableNameMap& outputs, const AttributeMap& attrs) { - return new OpType(type, inputs, outputs, attrs); - }; - op_info.grad_op_type_ = grad_op_type; - if (std::type_index(typeid(ProtoMakerType)) != - std::type_index(typeid(NOPMaker))) { - op_info.proto_ = new OpProto; - op_info.checker_ = new OpAttrChecker; - auto maker = ProtoMakerType(op_info.proto_, op_info.checker_); - maker.Validate(); - op_info.proto_->set_type(op_type); - PADDLE_ENFORCE( - op_info.proto_->IsInitialized(), - "Fail to initialize %s's OpProto, because %s is not initialized", - op_type, op_info.proto_->InitializationErrorString()); - } else { - op_info.proto_ = nullptr; - op_info.checker_ = nullptr; - } - OpInfoMap::Instance().Insert(op_type, op_info); + OperatorRegistrar reg(op_type.c_str()); + reg.info.grad_op_type_ = grad_op_type; // register gradient op if (!grad_op_type.empty()) { - RegisterOp(grad_op_type, ""); + OperatorRegistrar grad_reg(grad_op_type.c_str()); } } diff --git a/paddle/framework/op_registry_test.cc b/paddle/framework/op_registry_test.cc index b6fc0409d5..f89f40b444 100644 --- a/paddle/framework/op_registry_test.cc +++ b/paddle/framework/op_registry_test.cc @@ -173,3 +173,14 @@ TEST(OpRegistry, CustomChecker) { int test_attr = op->Attr("test_attr"); ASSERT_EQ(test_attr, 4); } + +class CosineOpComplete : public paddle::framework::CosineOp { + public: + DEFINE_OP_CONSTRUCTOR(CosineOpComplete, paddle::framework::CosineOp); + DEFINE_OP_CLONE_METHOD(CosineOpComplete); +}; + +TEST(OperatorRegistrar, Test) { + using namespace paddle::framework; + OperatorRegistrar reg("cos"); +} \ No newline at end of file From af6f3c0423f6ffd6c5480eeee5d340a5aad3e487 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Sat, 30 Sep 2017 15:34:52 -0700 Subject: [PATCH 061/170] use float32 in cond_op --- python/paddle/v2/framework/tests/test_cond_op.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/v2/framework/tests/test_cond_op.py b/python/paddle/v2/framework/tests/test_cond_op.py index 3698ce9c8e..76323b5e10 100644 --- a/python/paddle/v2/framework/tests/test_cond_op.py +++ b/python/paddle/v2/framework/tests/test_cond_op.py @@ -15,7 +15,7 @@ class PySimpleCond(object): for i in range(1, 10, 2): array[i] = 0 self.cond = np.array(array) - self.x = np.ones(shape=(10, 1)) + self.x = np.ones(shape=(10, 1)).astype("float32") def forward(self): self.index_t = np.where(self.cond == 1) From e3a642e027e3c749e5405f26478e28887cab504a Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Sat, 30 Sep 2017 15:39:42 -0700 Subject: [PATCH 062/170] Extract BaseClass of grad_op_desc_maker and add some common method --- paddle/framework/details/op_registry.h | 6 +- paddle/framework/grad_op_desc_maker.h | 115 +++++++++++++++++++++++++ paddle/framework/op_desc.h | 22 ++++- paddle/framework/op_info.h | 8 +- 4 files changed, 139 insertions(+), 12 deletions(-) create mode 100644 paddle/framework/grad_op_desc_maker.h diff --git a/paddle/framework/details/op_registry.h b/paddle/framework/details/op_registry.h index d2516ccc1e..daa474e8c5 100644 --- a/paddle/framework/details/op_registry.h +++ b/paddle/framework/details/op_registry.h @@ -14,6 +14,7 @@ #pragma once +#include "paddle/framework/grad_op_desc_maker.h" #include "paddle/framework/op_info.h" #include "paddle/framework/op_proto_maker.h" #include "paddle/framework/operator.h" @@ -96,7 +97,10 @@ struct OpInfoFiller { template struct OpInfoFiller { void operator()(const char* op_type, OpInfo* info) const { - info->grad_op_maker_ = new T(); + info->grad_op_maker_ = [](const OpDescBind& fwd_op) { + T maker(fwd_op); + return maker(); + }; } }; } // namespace details diff --git a/paddle/framework/grad_op_desc_maker.h b/paddle/framework/grad_op_desc_maker.h new file mode 100644 index 0000000000..cb4d160bd0 --- /dev/null +++ b/paddle/framework/grad_op_desc_maker.h @@ -0,0 +1,115 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once +#include "paddle/framework/op_desc.h" +#include "paddle/framework/operator.h" + +namespace paddle { +namespace framework { + +class GradOpDescMakerBase { + public: + explicit GradOpDescMakerBase(const OpDescBind& fwd_op) : fwd_op_(fwd_op) {} + + virtual ~GradOpDescMakerBase() = default; + virtual std::vector operator()() const = 0; + + protected: + static std::vector ToGradNames( + const std::vector& var_names) { + std::vector ret_val; + ret_val.reserve(var_names.size()); + std::transform(var_names.begin(), var_names.end(), + std::back_inserter(ret_val), GradVarName); + return ret_val; + } + + std::vector InputGrad(const std::string& name) const { + return ToGradNames(fwd_op_.Input(name)); + } + + std::vector OutputGrad(const std::string& name) const { + return ToGradNames(fwd_op_.Output(name)); + } + + std::vector InputParamNames() const { + return this->fwd_op_.InputParamNames(); + } + + std::vector OutputParamNames() const { + return this->fwd_op_.OutputParamNames(); + } + + std::vector Input(const std::string& name) const { + return fwd_op_.Input(name); + } + + std::vector Output(const std::string& name) const { + return fwd_op_.Output(name); + } + + const std::unordered_map& Attrs() const { + return fwd_op_.GetAttrMap(); + } + + const Attribute& GetAttr(const std::string& name) const { + auto& map = fwd_op_.GetAttrMap(); + auto it = map.find(name); + PADDLE_ENFORCE(it != map.end(), "Cannot find attribute %s", name); + return it->second; + } + + std::string ForwardOpType() const { return this->fwd_op_.Type(); } + + private: + const OpDescBind& fwd_op_; +}; + +class SingleGradOpDescMaker : public GradOpDescMakerBase { + public: + std::vector operator()() const { return {this->Apply()}; } + + protected: + virtual OpDescBind Apply() const = 0; +}; + +class DefaultGradOpDescMaker : public SingleGradOpDescMaker { + protected: + virtual OpDescBind Apply() const { + OpDescBind grad; + grad.SetType(this->GradOpType()); + + for (auto& input_param : this->InputParamNames()) { + grad.SetInput(input_param, this->Input(input_param)); + grad.SetOutput(GradVarName(input_param), this->InputGrad(input_param)); + } + + for (auto& output_param : this->OutputParamNames()) { + grad.SetInput(output_param, this->Output(output_param)); + grad.SetInput(GradVarName(output_param), this->OutputGrad(output_param)); + } + + grad.SetAttrMap(this->Attrs()); + + return grad; + } + + virtual std::string GradOpType() const { + return this->ForwardOpType() + "_grad"; + } +}; + +} // namespace framework +} // namespace paddle diff --git a/paddle/framework/op_desc.h b/paddle/framework/op_desc.h index 0cf7d13971..851a305061 100644 --- a/paddle/framework/op_desc.h +++ b/paddle/framework/op_desc.h @@ -60,17 +60,31 @@ class OpDescBind { void SetBlockAttr(const std::string &name, BlockDescBind &block); - // Only be used in C++ - void SetAttrMap(const std::unordered_map &attr_map); - Attribute GetAttr(const std::string &name) const; int GetBlockAttr(const std::string &name) const; - // Only be used in C++ + // The following methods should only be used in C++ const std::unordered_map &GetAttrMap() const; + void SetAttrMap(const std::unordered_map &attr_map); + + std::vector InputParamNames() const { return MapKeys(inputs_); } + std::vector OutputParamNames() const { + return MapKeys(outputs_); + } + private: + template + static std::vector MapKeys(const MapType &map) { + std::vector ret_val; + ret_val.reserve(map.size()); + std::transform( + map.begin(), map.end(), ret_val.begin(), + [](const typename MapType::value_type &pair) { return pair.first; }); + return ret_val; + } + struct SetAttrDescVisitor : public boost::static_visitor { explicit SetAttrDescVisitor(OpDesc::Attr *attr) : attr_(attr) {} mutable OpDesc::Attr *attr_; diff --git a/paddle/framework/op_info.h b/paddle/framework/op_info.h index 6d1ee4dece..8149c0061a 100644 --- a/paddle/framework/op_info.h +++ b/paddle/framework/op_info.h @@ -29,16 +29,10 @@ using OpCreator = std::function; -class GradOpDescMakerBase { - public: - virtual ~GradOpDescMakerBase() = default; - virtual std::vector operator()(const OpDescBind&) const = 0; -}; - struct OpInfo { OpCreator creator_; std::string grad_op_type_; - GradOpDescMakerBase* grad_op_maker_{nullptr}; + std::function(const OpDescBind&)> grad_op_maker_; OpProto* proto_{nullptr}; OpAttrChecker* checker_{nullptr}; From c23af80afe4d577f8e7b0c1c4bdd2dd53d5377f1 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Sat, 30 Sep 2017 16:11:40 -0700 Subject: [PATCH 063/170] Change macro --- paddle/framework/grad_op_desc_maker.h | 4 ++ paddle/framework/op_registry.h | 82 +++++++++++++++------------ 2 files changed, 50 insertions(+), 36 deletions(-) diff --git a/paddle/framework/grad_op_desc_maker.h b/paddle/framework/grad_op_desc_maker.h index cb4d160bd0..672cd7dbaf 100644 --- a/paddle/framework/grad_op_desc_maker.h +++ b/paddle/framework/grad_op_desc_maker.h @@ -79,6 +79,7 @@ class GradOpDescMakerBase { class SingleGradOpDescMaker : public GradOpDescMakerBase { public: + using GradOpDescMakerBase::GradOpDescMakerBase; std::vector operator()() const { return {this->Apply()}; } protected: @@ -86,6 +87,9 @@ class SingleGradOpDescMaker : public GradOpDescMakerBase { }; class DefaultGradOpDescMaker : public SingleGradOpDescMaker { + public: + using SingleGradOpDescMaker::SingleGradOpDescMaker; + protected: virtual OpDescBind Apply() const { OpDescBind grad; diff --git a/paddle/framework/op_registry.h b/paddle/framework/op_registry.h index 4ee2c7d275..7db095369e 100644 --- a/paddle/framework/op_registry.h +++ b/paddle/framework/op_registry.h @@ -24,14 +24,27 @@ limitations under the License. */ #include "paddle/framework/details/op_registry.h" #include "paddle/framework/framework.pb.h" #include "paddle/framework/grad_op_builder.h" +#include "paddle/framework/grad_op_desc_maker.h" #include "paddle/framework/operator.h" #include "paddle/framework/scope.h" namespace paddle { namespace framework { +class Registrar { + public: + // In our design, various kinds of classes, e.g., operators and kernels, + // have their corresponding registry and registrar. The action of + // registration is in the constructor of a global registrar variable, which, + // however, are not used in the code that calls package framework, and would + // be removed from the generated binary file by the linker. To avoid such + // removal, we add Touch to all registrar classes and make USE_OP macros to + // call this method. So, as long as the callee code calls USE_OP, the global + // registrar variable won't be removed by the linker. + void Touch() {} +}; template -struct OperatorRegistrar { +struct OperatorRegistrar : public Registrar { explicit OperatorRegistrar(const char* op_type) : op_type(op_type) { PADDLE_ENFORCE(!OpInfoMap::Instance().Has(op_type), "'%s' is registered more than once.", op_type); @@ -70,19 +83,6 @@ class OpRegistry { static std::unique_ptr CreateGradOp(const OperatorBase& op); }; -class Registrar { - public: - // In our design, various kinds of classes, e.g., operators and kernels, - // have their corresponding registry and registrar. The action of - // registration is in the constructor of a global registrar variable, which, - // however, are not used in the code that calls package framework, and would - // be removed from the generated binary file by the linker. To avoid such - // removal, we add Touch to all registrar classes and make USE_OP macros to - // call this method. So, as long as the callee code calls USE_OP, the global - // registrar variable won't be removed by the linker. - void Touch() {} -}; - template class OpRegistrar : public Registrar { public: @@ -138,33 +138,43 @@ class OpKernelRegistrar : public Registrar { __test_global_namespace_##uniq_name##__>::value, \ msg) +#define VA_ARGS(...) , ##__VA_ARGS__ + +#define REGISTER_OPERATOR(op_type, op_class, ...) \ + STATIC_ASSERT_GLOBAL_NAMESPACE( \ + __reg_op__##op_type, \ + "REGISTER_OPERATOR must be called in global namespace"); \ + class _OpClass_##op_type##_ : public op_class { \ + public: \ + DEFINE_OP_CLONE_METHOD(_OpClass_##op_type##_); \ + DEFINE_OP_CONSTRUCTOR(_OpClass_##op_type##_, op_class); \ + }; \ + static ::paddle::framework::OperatorRegistrar<_OpClass_##op_type##_ VA_ARGS( \ + __VA_ARGS__)> \ + __op_registrar_##op_type##__(#op_type); \ + int TouchOpRegistrar_##op_type() { \ + __op_registrar_##op_type##__.Touch(); \ + return 0; \ + } + /** * Macro to register Operator. */ -#define REGISTER_OP(op_type, op_class, op_maker_class, grad_op_type, \ - grad_op_class) \ - STATIC_ASSERT_GLOBAL_NAMESPACE( \ - __reg_op__##op_type, "REGISTER_OP must be called in global namespace"); \ - class _OpClass_##op_type##_ : public op_class { \ - public: \ - DEFINE_OP_CLONE_METHOD(_OpClass_##op_type##_); \ - DEFINE_OP_CONSTRUCTOR(_OpClass_##op_type##_, op_class); \ - }; \ - class _OpGradClass_##op_type##_ : public grad_op_class { \ - public: \ - DEFINE_OP_CLONE_METHOD(_OpGradClass_##op_type##_); \ - DEFINE_OP_CONSTRUCTOR(_OpGradClass_##op_type##_, grad_op_class); \ - }; \ - static ::paddle::framework::OpRegistrar< \ - _OpClass_##op_type##_, op_maker_class, _OpGradClass_##op_type##_> \ - __op_registrar_##op_type##__(#op_type, #grad_op_type); \ - int TouchOpRegistrar_##op_type() { \ - __op_registrar_##op_type##__.Touch(); \ - return 0; \ - } +#define REGISTER_OP(op_type, op_class, op_maker_class, grad_op_type, \ + grad_op_class) \ + REGISTER_OPERATOR(grad_op_type, grad_op_class); \ + class _GradOpDescMaker_##grad_op_type##_ \ + : public ::paddle::framework::DefaultGradOpDescMaker { \ + using ::paddle::framework::DefaultGradOpDescMaker::DefaultGradOpDescMaker; \ + \ + protected: \ + virtual std::string GradOpType() const { return #grad_op_type; } \ + }; \ + REGISTER_OPERATOR(op_type, op_class, _GradOpDescMaker_##grad_op_type##_, \ + op_maker_class) #define REGISTER_OP_WITHOUT_GRADIENT(op_type, op_class, op_maker_class) \ - REGISTER_OP(op_type, op_class, op_maker_class, , ::paddle::framework::NOP) + REGISTER_OPERATOR(op_type, op_class, op_maker_class) /** * Macro to register OperatorKernel. From d64bedf638d66cc4fedb63bcfd389a1058359798 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Sat, 30 Sep 2017 16:44:16 -0700 Subject: [PATCH 064/170] Stash --- paddle/framework/backward.cc | 3 + paddle/framework/backward_test.cc | 31 ++-- paddle/framework/grad_op_builder.cc | 97 ----------- paddle/framework/grad_op_builder.h | 28 ---- paddle/framework/grad_op_builder_test.cc | 201 ----------------------- paddle/framework/op_desc.h | 12 ++ paddle/framework/op_registry.cc | 5 - paddle/framework/op_registry.h | 2 - 8 files changed, 25 insertions(+), 354 deletions(-) delete mode 100644 paddle/framework/grad_op_builder.cc delete mode 100644 paddle/framework/grad_op_builder.h delete mode 100644 paddle/framework/grad_op_builder_test.cc diff --git a/paddle/framework/backward.cc b/paddle/framework/backward.cc index 0ec18de5b8..ab2567a25c 100644 --- a/paddle/framework/backward.cc +++ b/paddle/framework/backward.cc @@ -154,6 +154,9 @@ static std::unique_ptr BackwardRecursive( net->InsertOp(pos.first + 1, std::move(pos.second)); } } else { + OpDescBind fwd_desc; + fwd_desc.SetInput(forwardOp.Inputs()); + std::unique_ptr grad_op(OpRegistry::CreateGradOp(forwardOp)); ForEachVarName(grad_op->Inputs(), [&no_grad_names, &net, &grad_op]( diff --git a/paddle/framework/backward_test.cc b/paddle/framework/backward_test.cc index 6932f5b989..28fc6f9ced 100644 --- a/paddle/framework/backward_test.cc +++ b/paddle/framework/backward_test.cc @@ -159,16 +159,16 @@ REGISTER_OP_WITHOUT_GRADIENT(fc, f::FcOp, f::FcOpMaker); REGISTER_OP(many_output_op, f::NOP, f::ManyOutputOpMaker, many_output_op_grad, f::NOP); -TEST(Backward, simple_op_grad) { - auto fwd = f::OpRegistry::CreateOp( - "rowwise_add", {{"X", {"x"}}, {"b", {"b"}}}, {{"Out", {"out"}}}, {}); - ASSERT_NE(fwd, nullptr); - auto gop = f::OpRegistry::CreateGradOp(*fwd); - ASSERT_EQ(1UL, gop->Inputs().size()); - ASSERT_EQ("rowwise_add_grad", gop->Type()); - ASSERT_EQ(f::GradVarName("x"), gop->Output(f::GradVarName("X"))); - ASSERT_EQ(f::GradVarName("b"), gop->Output(f::GradVarName("b"))); -} +// TEST(Backward, simple_op_grad) { +// auto fwd = f::OpRegistry::CreateOp( +// "rowwise_add", {{"X", {"x"}}, {"b", {"b"}}}, {{"Out", {"out"}}}, {}); +// ASSERT_NE(fwd, nullptr); +// auto gop = f::OpRegistry::CreateGradOp(*fwd); +// ASSERT_EQ(1UL, gop->Inputs().size()); +// ASSERT_EQ("rowwise_add_grad", gop->Type()); +// ASSERT_EQ(f::GradVarName("x"), gop->Output(f::GradVarName("X"))); +// ASSERT_EQ(f::GradVarName("b"), gop->Output(f::GradVarName("b"))); +//} TEST(Backward, simple_op_not_need_grad) { auto fwd = f::OpRegistry::CreateOp( @@ -286,17 +286,6 @@ TEST(Backward, net_shared_weight) { ASSERT_EQ("add", bwd_net->ops_[2]->Type()); } -TEST(Backward, op_register_grad_not_for_network) { - auto fwd = - f::OpRegistry::CreateOp("fc", {{"X", {"x"}}, {"W", {"w"}}, {"b", {"b"}}}, - {{"mul_result", {"mul_out"}}, - {"add_result", {"add_out"}}, - {"Out", {"out1"}}}, - {{"temporary_index", std::vector{0, 1}}}); - - ASSERT_THROW(f::OpRegistry::CreateGradOp(*fwd), EnforceNotMet); -} - TEST(Backward, op_all_input_are_not_need) { auto fwd = f::OpRegistry::CreateOp( "rowwise_add", {{"X", {"x"}}, {"b", {"b"}}}, {{"Out", {"out"}}}, {}); diff --git a/paddle/framework/grad_op_builder.cc b/paddle/framework/grad_op_builder.cc deleted file mode 100644 index 3661ce41be..0000000000 --- a/paddle/framework/grad_op_builder.cc +++ /dev/null @@ -1,97 +0,0 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOpArgType::OUT WARRANTIES OR CONDITIONS OF ANY KOpArgType::IND, either -express or implied. See the License for the specific language governing -permissions and limitations under the License. */ - -#include "paddle/framework/grad_op_builder.h" -#include "paddle/framework/op_registry.h" - -namespace paddle { -namespace framework { -enum class OpArgType { IN, OUT }; - -static void TransOpArg(const OperatorBase* src_op, const OpArgType& src_type, - bool is_grad, VariableNameMap* vars) { - const auto& src_inout = - src_type == OpArgType::IN ? src_op->Inputs() : src_op->Outputs(); - auto& dst_inout = *vars; - auto& proto = OpInfoMap::Instance().Get(src_op->Type()).Proto(); - const auto& src_arg_list = - src_type == OpArgType::IN ? proto.inputs() : proto.outputs(); - for (const auto& arg : src_arg_list) { - if (arg.not_in_gradient() && !is_grad) continue; - const std::string src_name = arg.name(); - std::string dst_name = is_grad ? GradVarName(src_name) : src_name; - dst_inout[dst_name].reserve(src_inout.at(src_name).size()); - for (auto& var_name : src_inout.at(src_name)) { - std::string s = is_grad ? GradVarName(var_name) : var_name; - dst_inout[dst_name].emplace_back(s); - } - } -} - -OperatorBase* BuildGradOp(const OperatorBase* op) { - auto& info = OpInfoMap::Instance().Get(op->Type()); - PADDLE_ENFORCE(info.HasGradientOp()); - - VariableNameMap inputs; - VariableNameMap outputs; - TransOpArg(op, OpArgType::IN, false, &inputs); // I - TransOpArg(op, OpArgType::OUT, false, &inputs); // O - TransOpArg(op, OpArgType::OUT, true, &inputs); // OG - TransOpArg(op, OpArgType::IN, true, &outputs); // IG - - auto& grad_info = OpInfoMap::Instance().Get(info.grad_op_type_); - return grad_info.Creator()(info.grad_op_type_, inputs, outputs, op->Attrs()); -} - -static void TransOpDescArg(const OpDescBind* src_op, const OpArgType& src_type, - bool is_grad, OpDescBind* dst_op, - const OpArgType& dst_type) { - PADDLE_ENFORCE(dst_op != nullptr, - "Protobuf desc of gradient op must be initialized first."); - const auto& proto = OpInfoMap::Instance().Get(src_op->Type()).Proto(); - const auto& src_arg_list = - src_type == OpArgType::IN ? proto.inputs() : proto.outputs(); - for (const auto& arg : src_arg_list) { - if (arg.not_in_gradient() && !is_grad) continue; - const std::string src_name = arg.name(); - std::vector vars = src_type == OpArgType::IN - ? src_op->Input(src_name) - : src_op->Output(src_name); - if (is_grad) { - for (std::string& var : vars) { - var = GradVarName(var); - } - } - std::string dst_name = is_grad ? GradVarName(src_name) : src_name; - dst_type == OpArgType::IN ? dst_op->SetInput(dst_name, vars) - : dst_op->SetOutput(dst_name, vars); - } -} - -void CompleteGradOpDesc(const OpDescBind* forw_op, OpDescBind* grad_op) { - auto& info = OpInfoMap::Instance().Get(forw_op->Type()); - PADDLE_ENFORCE(info.HasGradientOp()); - - grad_op->SetType(info.grad_op_type_); - - TransOpDescArg(forw_op, OpArgType::IN, false, grad_op, OpArgType::IN); - TransOpDescArg(forw_op, OpArgType::OUT, false, grad_op, OpArgType::IN); - TransOpDescArg(forw_op, OpArgType::OUT, true, grad_op, OpArgType::IN); - TransOpDescArg(forw_op, OpArgType::IN, true, grad_op, OpArgType::OUT); - - grad_op->SetAttrMap(forw_op->GetAttrMap()); -} - -} // namespace framework -} // namespace paddle diff --git a/paddle/framework/grad_op_builder.h b/paddle/framework/grad_op_builder.h deleted file mode 100644 index b601406061..0000000000 --- a/paddle/framework/grad_op_builder.h +++ /dev/null @@ -1,28 +0,0 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ - -#pragma once - -#include "paddle/framework/op_desc.h" -#include "paddle/framework/operator.h" - -namespace paddle { -namespace framework { - -OperatorBase* BuildGradOp(const OperatorBase* op); - -void CompleteGradOpDesc(const OpDescBind* forw_op, OpDescBind* grad_op); - -} // namespace framework -} // namespace paddle diff --git a/paddle/framework/grad_op_builder_test.cc b/paddle/framework/grad_op_builder_test.cc deleted file mode 100644 index d09892f81b..0000000000 --- a/paddle/framework/grad_op_builder_test.cc +++ /dev/null @@ -1,201 +0,0 @@ -#include "paddle/framework/grad_op_builder.h" -#include -#include "paddle/framework/op_registry.h" -#include "paddle/framework/operator.h" - -USE_OP(add); - -namespace paddle { -namespace framework { - -class MutiInOutOpMaker : public OpProtoAndCheckerMaker { - public: - MutiInOutOpMaker(OpProto *proto, OpAttrChecker *op_checker) - : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("In1", "a single input"); - AddInput("In2_mult", "a multiple input").AsDuplicable(); - AddInput("In3", "another single input"); - AddOutput("Out1", "a single output"); - AddOutput("Out2_mult", "a multiple output").AsDuplicable(); - AddComment("test op with multiple inputs and outputs"); - } -}; - -class IOIgnoredOpMaker : public OpProtoAndCheckerMaker { - public: - IOIgnoredOpMaker(OpProto *proto, OpAttrChecker *op_checker) - : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("In1", "a single input"); - AddInput("In2_mult", "a multiple input").AsDuplicable().NotInGradient(); - AddInput("In3_mult", "another multiple input").AsDuplicable(); - AddOutput("Out1_mult", "a multiple output").AsDuplicable(); - AddOutput("Out2", "a single output").NotInGradient(); - AddComment("op with inputs and outputs ignored in gradient calculating"); - } -}; - -} // namespace framework -} // namespace paddle - -namespace f = paddle::framework; - -TEST(GradOpBuilder, AddTwo) { - std::shared_ptr add_op(f::OpRegistry::CreateOp( - "add", {{"X", {"x"}}, {"Y", {"y"}}}, {{"Out", {"out"}}}, {})); - std::shared_ptr grad_add_op = - f::OpRegistry::CreateGradOp(*add_op); - EXPECT_EQ(grad_add_op->Inputs().size(), 4UL); - EXPECT_EQ(grad_add_op->Outputs().size(), 2UL); - EXPECT_EQ(grad_add_op->Input("X"), "x"); - EXPECT_EQ(grad_add_op->Input("Y"), "y"); - EXPECT_EQ(grad_add_op->Input("Out"), "out"); - EXPECT_EQ(grad_add_op->Input(f::GradVarName("Out")), f::GradVarName("out")); - EXPECT_EQ(grad_add_op->Output(f::GradVarName("X")), f::GradVarName("x")); - EXPECT_EQ(grad_add_op->Output(f::GradVarName("Y")), f::GradVarName("y")); -} - -REGISTER_OP(mult_io, f::NOP, f::MutiInOutOpMaker, mult_io_grad, f::NOP); -REGISTER_OP(io_ignored, f::NOP, f::IOIgnoredOpMaker, io_ignored_grad, f::NOP); - -TEST(GradOpBuilder, MutiInOut) { - std::shared_ptr test_op(f::OpRegistry::CreateOp( - "mult_io", {{"In1", {"in1"}}, - {"In2_mult", {"in2_1", "in2_2", "in2_3"}}, - {"In3", {"in3"}}}, - {{"Out1", {"out1"}}, {"Out2_mult", {"out2_1", "out2_2"}}}, {})); - std::shared_ptr grad_test_op = - f::OpRegistry::CreateGradOp(*test_op); - - ASSERT_EQ(grad_test_op->Inputs().size(), 3UL + 2UL + 2UL); - EXPECT_EQ(grad_test_op->Input("In1"), "in1"); - EXPECT_EQ(grad_test_op->Inputs("In2_mult"), - std::vector({"in2_1", "in2_2", "in2_3"})); - EXPECT_EQ(grad_test_op->Input("In3"), "in3"); - EXPECT_EQ(grad_test_op->Input("Out1"), "out1"); - EXPECT_EQ(grad_test_op->Inputs("Out2_mult"), - std::vector({"out2_1", "out2_2"})); - EXPECT_EQ(grad_test_op->Input(f::GradVarName("Out1")), - f::GradVarName("out1")); - EXPECT_EQ(grad_test_op->Inputs(f::GradVarName("Out2_mult")), - std::vector( - {f::GradVarName("out2_1"), f::GradVarName("out2_2")})); - - ASSERT_EQ(grad_test_op->Outputs().size(), 3UL); - EXPECT_EQ(grad_test_op->Output(f::GradVarName("In1")), f::GradVarName("in1")); - EXPECT_EQ(grad_test_op->Outputs(f::GradVarName("In2_mult")), - std::vector({f::GradVarName("in2_1"), - f::GradVarName("in2_2"), - f::GradVarName("in2_3")})); - EXPECT_EQ(grad_test_op->Output(f::GradVarName("In3")), f::GradVarName("in3")); -} - -TEST(GradOpBuilder, IOIgnoredInGradient) { - std::shared_ptr test_op(f::OpRegistry::CreateOp( - "io_ignored", {{"In1", {"in1"}}, - {"In2_mult", {"in2_1", "in2_2"}}, - {"In3_mult", {"in3_1", "in3_2"}}}, - {{"Out1_mult", {"out1_1", "out1_2"}}, {"Out2", {"out2"}}}, {})); - std::shared_ptr grad_test_op = - f::OpRegistry::CreateGradOp(*test_op); - - // 'In2' and 'Out2' are ignored in gradient calculating - ASSERT_EQ(grad_test_op->Inputs().size(), 2UL + 1UL + 2UL); - EXPECT_EQ(grad_test_op->Input("In1"), "in1"); - EXPECT_EQ(grad_test_op->Inputs("In3_mult"), - std::vector({"in3_1", "in3_2"})); - EXPECT_EQ(grad_test_op->Inputs("Out1_mult"), - std::vector({"out1_1", "out1_2"})); - EXPECT_EQ(grad_test_op->Inputs(f::GradVarName("Out1_mult")), - std::vector( - {f::GradVarName("out1_1"), f::GradVarName("out1_2")})); - EXPECT_EQ(grad_test_op->Input(f::GradVarName("Out2")), - f::GradVarName("out2")); - - ASSERT_EQ(grad_test_op->Outputs().size(), 3UL); - EXPECT_EQ(grad_test_op->Output(f::GradVarName("In1")), f::GradVarName("in1")); - EXPECT_EQ(grad_test_op->Outputs(f::GradVarName("In2_mult")), - std::vector( - {f::GradVarName("in2_1"), f::GradVarName("in2_2")})); - EXPECT_EQ(grad_test_op->Outputs(f::GradVarName("In3_mult")), - std::vector( - {f::GradVarName("in3_1"), f::GradVarName("in3_2")})); -} - -TEST(GradOpDescBuilder, MutiInOut) { - f::OpDescBind *forw_op = new f::OpDescBind(); - forw_op->SetType("mult_io"); - forw_op->SetInput("In1", {"in1"}); - forw_op->SetInput("In2_mult", {"in2_1", "in2_2", "in2_3"}); - forw_op->SetInput("In3", {"in3"}); - forw_op->SetOutput("Out1", {"out1"}); - forw_op->SetOutput("Out2_mult", {"out2_1", "out2_2"}); - - f::OpDescBind *grad_op = new f::OpDescBind(); - f::CompleteGradOpDesc(forw_op, grad_op); - - EXPECT_EQ(grad_op->Type(), "mult_io_grad"); - ASSERT_EQ(grad_op->InputNames().size(), 3UL + 2UL + 2UL); - EXPECT_EQ(grad_op->Input("In1"), std::vector({"in1"})); - EXPECT_EQ(grad_op->Input("In2_mult"), - std::vector({"in2_1", "in2_2", "in2_3"})); - EXPECT_EQ(grad_op->Input("In3"), std::vector({"in3"})); - EXPECT_EQ(grad_op->Input("Out1"), std::vector({"out1"})); - EXPECT_EQ(grad_op->Input("Out2_mult"), - std::vector({"out2_1", "out2_2"})); - EXPECT_EQ(grad_op->Input(f::GradVarName("Out1")), - std::vector({f::GradVarName("out1")})); - EXPECT_EQ(grad_op->Input(f::GradVarName("Out2_mult")), - std::vector( - {f::GradVarName("out2_1"), f::GradVarName("out2_2")})); - - ASSERT_EQ(grad_op->OutputNames().size(), 3UL); - EXPECT_EQ(grad_op->Output(f::GradVarName("In1")), - std::vector({f::GradVarName("in1")})); - EXPECT_EQ(grad_op->Output(f::GradVarName("In2_mult")), - std::vector({f::GradVarName("in2_1"), - f::GradVarName("in2_2"), - f::GradVarName("in2_3")})); - EXPECT_EQ(grad_op->Output(f::GradVarName("In3")), - std::vector({f::GradVarName("in3")})); - delete forw_op; - delete grad_op; -} - -TEST(GradOpDescBuilder, IOIgnoredInGradient) { - f::OpDescBind *forw_op = new f::OpDescBind(); - forw_op->SetType("io_ignored"); - forw_op->SetInput("In1", {"in1"}); - forw_op->SetInput("In2_mult", {"in2_1", "in2_2"}); - forw_op->SetInput("In3_mult", {"in3_1", "in3_2"}); - forw_op->SetOutput("Out1_mult", {"out1_1", "out1_2"}); - forw_op->SetOutput("Out2", {"out2"}); - - f::OpDescBind *grad_op = new f::OpDescBind(); - f::CompleteGradOpDesc(forw_op, grad_op); - - EXPECT_EQ(grad_op->Type(), "io_ignored_grad"); - // 'In2' and 'Out2' are ignored in gradient calculating - ASSERT_EQ(grad_op->InputNames().size(), 2UL + 1UL + 2UL); - EXPECT_EQ(grad_op->Input("In1"), std::vector({"in1"})); - EXPECT_EQ(grad_op->Input("In3_mult"), - std::vector({"in3_1", "in3_2"})); - EXPECT_EQ(grad_op->Input("Out1_mult"), - std::vector({"out1_1", "out1_2"})); - EXPECT_EQ(grad_op->Input(f::GradVarName("Out1_mult")), - std::vector( - {f::GradVarName("out1_1"), f::GradVarName("out1_2")})); - EXPECT_EQ(grad_op->Input(f::GradVarName("Out2")), - std::vector({f::GradVarName("out2")})); - - ASSERT_EQ(grad_op->OutputNames().size(), 3UL); - EXPECT_EQ(grad_op->Output(f::GradVarName("In1")), - std::vector({f::GradVarName("in1")})); - EXPECT_EQ(grad_op->Output(f::GradVarName("In2_mult")), - std::vector( - {f::GradVarName("in2_1"), f::GradVarName("in2_2")})); - EXPECT_EQ(grad_op->Output(f::GradVarName("In3_mult")), - std::vector( - {f::GradVarName("in3_1"), f::GradVarName("in3_2")})); - delete forw_op; - delete grad_op; -} \ No newline at end of file diff --git a/paddle/framework/op_desc.h b/paddle/framework/op_desc.h index 851a305061..4b001fb964 100644 --- a/paddle/framework/op_desc.h +++ b/paddle/framework/op_desc.h @@ -74,6 +74,18 @@ class OpDescBind { return MapKeys(outputs_); } + void SetInput( + const std::unordered_map> &input) { + this->inputs_ = input; + this->need_update_ = true; + } + + void SetOutput( + const std::unordered_map> &output) { + this->outputs_ = output; + this->need_update_ = true; + } + private: template static std::vector MapKeys(const MapType &map) { diff --git a/paddle/framework/op_registry.cc b/paddle/framework/op_registry.cc index b0e85dd49f..0a2b6fd582 100644 --- a/paddle/framework/op_registry.cc +++ b/paddle/framework/op_registry.cc @@ -52,10 +52,5 @@ std::unique_ptr OpRegistry::CreateOp(const OpDesc& op_desc) { return CreateOp(op_desc.type(), inputs, outputs, attrs); } -std::unique_ptr OpRegistry::CreateGradOp(const OperatorBase& op) { - PADDLE_ENFORCE(!op.IsNetOp(), "Use framework::Backward to get backward ops"); - return std::unique_ptr(BuildGradOp(&op)); -} - } // namespace framework } // namespace paddle diff --git a/paddle/framework/op_registry.h b/paddle/framework/op_registry.h index 7db095369e..0f377f34cb 100644 --- a/paddle/framework/op_registry.h +++ b/paddle/framework/op_registry.h @@ -79,8 +79,6 @@ class OpRegistry { AttributeMap attrs); static std::unique_ptr CreateOp(const OpDesc& op_desc); - - static std::unique_ptr CreateGradOp(const OperatorBase& op); }; template From 8fd845e0fab40acc9539c4109feb3ed411f4dc8b Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Sat, 30 Sep 2017 16:55:40 -0700 Subject: [PATCH 065/170] Unify Map in OpDescBind --- paddle/framework/op_desc.cc | 27 ++++++++++++++++++++++++++- paddle/framework/op_desc.h | 37 ++++++------------------------------- paddle/platform/enforce.h | 4 ++-- 3 files changed, 34 insertions(+), 34 deletions(-) diff --git a/paddle/framework/op_desc.cc b/paddle/framework/op_desc.cc index 0c12c55dc0..33a064890c 100644 --- a/paddle/framework/op_desc.cc +++ b/paddle/framework/op_desc.cc @@ -112,6 +112,30 @@ const std::unordered_map &OpDescBind::GetAttrMap() return attrs_; } +struct SetAttrDescVisitor : public boost::static_visitor { + explicit SetAttrDescVisitor(OpDesc::Attr *attr) : attr_(attr) {} + mutable OpDesc::Attr *attr_; + void operator()(int v) const { attr_->set_i(v); } + void operator()(float v) const { attr_->set_f(v); } + void operator()(const std::string &v) const { attr_->set_s(v); } + void operator()(bool b) const { attr_->set_b(b); } + + void operator()(const std::vector &v) const { + VectorToRepeated(v, attr_->mutable_ints()); + } + void operator()(const std::vector &v) const { + VectorToRepeated(v, attr_->mutable_floats()); + } + void operator()(const std::vector &v) const { + VectorToRepeated(v, attr_->mutable_strings()); + } + void operator()(const std::vector &v) const { + VectorToRepeated(v, attr_->mutable_bools()); + } + void operator()(BlockDesc *desc) const { attr_->set_block_idx(desc->idx()); } + void operator()(boost::blank) const { PADDLE_THROW("Unexpected branch"); } +}; + void OpDescBind::Sync() { if (need_update_) { this->op_desc_.mutable_inputs()->Clear(); @@ -134,7 +158,8 @@ void OpDescBind::Sync() { attr_desc->set_name(attr.first); attr_desc->set_type( static_cast(attr.second.which() - 1)); - boost::apply_visitor(SetAttrDescVisitor(attr_desc), attr.second); + SetAttrDescVisitor visitor(attr_desc); + boost::apply_visitor(visitor, attr.second); } need_update_ = false; diff --git a/paddle/framework/op_desc.h b/paddle/framework/op_desc.h index 0cf7d13971..e03b4d067f 100644 --- a/paddle/framework/op_desc.h +++ b/paddle/framework/op_desc.h @@ -17,6 +17,7 @@ limitations under the License. */ #include #include #include "paddle/framework/attribute.h" +#include "paddle/framework/op_info.h" #include "paddle/framework/var_desc.h" namespace paddle { @@ -61,48 +62,22 @@ class OpDescBind { void SetBlockAttr(const std::string &name, BlockDescBind &block); // Only be used in C++ - void SetAttrMap(const std::unordered_map &attr_map); + void SetAttrMap(const AttributeMap &attr_map); Attribute GetAttr(const std::string &name) const; int GetBlockAttr(const std::string &name) const; // Only be used in C++ - const std::unordered_map &GetAttrMap() const; + const AttributeMap &GetAttrMap() const; private: - struct SetAttrDescVisitor : public boost::static_visitor { - explicit SetAttrDescVisitor(OpDesc::Attr *attr) : attr_(attr) {} - mutable OpDesc::Attr *attr_; - void operator()(int v) const { attr_->set_i(v); } - void operator()(float v) const { attr_->set_f(v); } - void operator()(const std::string &v) const { attr_->set_s(v); } - void operator()(bool b) const { attr_->set_b(b); } - - void operator()(const std::vector &v) const { - VectorToRepeated(v, attr_->mutable_ints()); - } - void operator()(const std::vector &v) const { - VectorToRepeated(v, attr_->mutable_floats()); - } - void operator()(const std::vector &v) const { - VectorToRepeated(v, attr_->mutable_strings()); - } - void operator()(const std::vector &v) const { - VectorToRepeated(v, attr_->mutable_bools()); - } - void operator()(BlockDesc *desc) const { - attr_->set_block_idx(desc->idx()); - } - void operator()(boost::blank) const { PADDLE_THROW("Unexpected branch"); } - }; - void Sync(); OpDesc op_desc_; - std::unordered_map> inputs_; - std::unordered_map> outputs_; - std::unordered_map attrs_; + VariableNameMap inputs_; + VariableNameMap outputs_; + AttributeMap attrs_; // need_update_ indicate there some local changes not be synchronized. If // local changes should be synchronized, need_update_ should be set to true. diff --git a/paddle/platform/enforce.h b/paddle/platform/enforce.h index b523ef03c0..52bd23039b 100644 --- a/paddle/platform/enforce.h +++ b/paddle/platform/enforce.h @@ -185,7 +185,7 @@ inline void throw_on_error(T e) { std::make_exception_ptr( \ std::runtime_error(paddle::string::Sprintf(__VA_ARGS__))), \ __FILE__, __LINE__); \ - } while (0) + } while (false) #define PADDLE_ENFORCE(...) \ do { \ @@ -195,7 +195,7 @@ inline void throw_on_error(T e) { throw ::paddle::platform::EnforceNotMet(std::current_exception(), \ __FILE__, __LINE__); \ } \ - } while (0) + } while (false) /* * Some enforce helpers here, usage: From 87efa600df84db136ae743dabd294e54b2d1805a Mon Sep 17 00:00:00 2001 From: Qiao Longfei Date: Sat, 30 Sep 2017 23:16:11 -0700 Subject: [PATCH 066/170] add some check to operator.run (#4544) * fix cond_op_test and add some check to operator.run * tmp * optimize kernel check --- paddle/framework/operator.cc | 7 +++++++ paddle/framework/operator.h | 25 ++++++++++++++++++++++--- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/paddle/framework/operator.cc b/paddle/framework/operator.cc index 8b5560ffa1..1012a30b0a 100644 --- a/paddle/framework/operator.cc +++ b/paddle/framework/operator.cc @@ -245,5 +245,12 @@ std::vector InferShapeContext::MultiOutput( return res; } +std::ostream& operator<<(std::ostream& os, + const OperatorWithKernel::OpKernelKey& kernel_key) { + os << "place[" << kernel_key.place_ << "]:data_type[" << kernel_key.data_type_ + << "]"; + return os; +} + } // namespace framework } // namespace paddle diff --git a/paddle/framework/operator.h b/paddle/framework/operator.h index 0af527c88c..73e53a4176 100644 --- a/paddle/framework/operator.h +++ b/paddle/framework/operator.h @@ -478,9 +478,25 @@ class OperatorWithKernel : public OperatorBase { this->InferShape(&infer_shape_ctx); ExecutionContext ctx(*this, scope, dev_ctx); - auto& opKernel = AllOpKernels().at(type_).at( - OpKernelKey(IndicateDataType(ctx), dev_ctx)); - opKernel->Compute(ctx); + + // check if op[type] has kernel registered. + auto& all_op_kernels = AllOpKernels(); + auto kernels_iter = all_op_kernels.find(type_); + if (kernels_iter == all_op_kernels.end()) { + PADDLE_THROW("op[%s] has no kernel", type_); + } + + // check if op[type] have kernel for kernel_key + OpKernelMap& kernels = kernels_iter->second; + auto kernel_key = OpKernelKey(IndicateDataType(ctx), dev_ctx); + auto kernel_iter = kernels.find(kernel_key); + + if (kernel_iter == kernels.end()) { + PADDLE_THROW("op[%s] has no kernel with kernel_key[%s]", type_, + kernel_key); + } + + kernel_iter->second->Compute(ctx); } static std::unordered_map& @@ -529,5 +545,8 @@ class OperatorWithKernel : public OperatorBase { } }; +std::ostream& operator<<(std::ostream& os, + const OperatorWithKernel::OpKernelKey& kernel_key); + } // namespace framework } // namespace paddle From 3e99b166ba147b8d954332a9be882bee25ca6591 Mon Sep 17 00:00:00 2001 From: dzhwinter Date: Sun, 1 Oct 2017 08:29:09 +0000 Subject: [PATCH 067/170] add generic add operator --- paddle/framework/backward.cc | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/paddle/framework/backward.cc b/paddle/framework/backward.cc index 0ec18de5b8..c625c0caf7 100644 --- a/paddle/framework/backward.cc +++ b/paddle/framework/backward.cc @@ -141,9 +141,35 @@ static std::unique_ptr BackwardRecursive( net->ops_[op_offset]->Rename(name, dup_outputs.back()); } // collect all the offset to append `add` op for each alias - insert_position.push_back( - {dup_op.back(), OpRegistry::CreateOp("add", {{"X", {dup_outputs}}}, - {{"Out", {name}}}, {})}); + // + // one variable is shared between multiple operators. + // insert add operator one by one, then add it to output + if (dup_outputs.size() == 2) { + insert_position.push_back( + {dup_op.back(), + OpRegistry::CreateOp( + "add", {{"X", {dup_outputs[0]}}, {"Y", {dup_outputs[1]}}}, + {{"Out", {name}}}, {})}); + } else { + for (size_t output_idx = 0; output_idx < dup_outputs.size() - 1; + ++output_idx) { + auto insert_add_x = dup_outputs[output_idx]; + auto insert_add_y = dup_outputs[output_idx]; + auto insert_add_out = name + "@SHARED@" + std::to_string(output_idx); + // first add op inserted + if (output_idx == dup_outputs.size() - 1) { + insert_add_out = name; + } + if (output_idx != 0) { + insert_add_y = name + "@SHARED@" + std::to_string(output_idx); + } + insert_position.push_back( + {dup_op.back(), + OpRegistry::CreateOp( + "add", {{"X", {insert_add_x}}, {"Y", {insert_add_y}}}, + {{"Out", {insert_add_out}}}, {})}); + } + } } // make sure the inserted `add` ops follow the BFS order. From c08635898f3a57cdf19c45f12b1aac28d864c73e Mon Sep 17 00:00:00 2001 From: dzhwinter Date: Sun, 1 Oct 2017 08:36:04 +0000 Subject: [PATCH 068/170] fix typo --- paddle/framework/backward.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/framework/backward.cc b/paddle/framework/backward.cc index c625c0caf7..b850939040 100644 --- a/paddle/framework/backward.cc +++ b/paddle/framework/backward.cc @@ -161,7 +161,7 @@ static std::unique_ptr BackwardRecursive( insert_add_out = name; } if (output_idx != 0) { - insert_add_y = name + "@SHARED@" + std::to_string(output_idx); + insert_add_y = name + "@SHARED@" + std::to_string(output_idx-1); } insert_position.push_back( {dup_op.back(), From e9a9dd6d4d5fb6428917cffb678cfa0b582fc018 Mon Sep 17 00:00:00 2001 From: dongzhihong Date: Sun, 1 Oct 2017 09:18:21 -0700 Subject: [PATCH 069/170] relauch ci --- paddle/framework/backward.cc | 36 ++++++++++++++---------------------- 1 file changed, 14 insertions(+), 22 deletions(-) diff --git a/paddle/framework/backward.cc b/paddle/framework/backward.cc index b850939040..fbacfeed94 100644 --- a/paddle/framework/backward.cc +++ b/paddle/framework/backward.cc @@ -144,31 +144,23 @@ static std::unique_ptr BackwardRecursive( // // one variable is shared between multiple operators. // insert add operator one by one, then add it to output - if (dup_outputs.size() == 2) { + for (size_t output_idx = 0; output_idx < dup_outputs.size() - 1; + ++output_idx) { + auto insert_add_x = dup_outputs[output_idx]; + auto insert_add_y = dup_outputs[output_idx]; + auto insert_add_out = name + "@SHARED@" + std::to_string(output_idx); + // first add op inserted + if (output_idx == dup_outputs.size() - 2) { + insert_add_out = name; + } + if (output_idx != 0) { + insert_add_y = name + "@SHARED@" + std::to_string(output_idx - 1); + } insert_position.push_back( {dup_op.back(), OpRegistry::CreateOp( - "add", {{"X", {dup_outputs[0]}}, {"Y", {dup_outputs[1]}}}, - {{"Out", {name}}}, {})}); - } else { - for (size_t output_idx = 0; output_idx < dup_outputs.size() - 1; - ++output_idx) { - auto insert_add_x = dup_outputs[output_idx]; - auto insert_add_y = dup_outputs[output_idx]; - auto insert_add_out = name + "@SHARED@" + std::to_string(output_idx); - // first add op inserted - if (output_idx == dup_outputs.size() - 1) { - insert_add_out = name; - } - if (output_idx != 0) { - insert_add_y = name + "@SHARED@" + std::to_string(output_idx-1); - } - insert_position.push_back( - {dup_op.back(), - OpRegistry::CreateOp( - "add", {{"X", {insert_add_x}}, {"Y", {insert_add_y}}}, - {{"Out", {insert_add_out}}}, {})}); - } + "add", {{"X", {insert_add_x}}, {"Y", {insert_add_y}}}, + {{"Out", {insert_add_out}}}, {})}); } } From 7163dd0413d5b99261ff95e0fab28a09f8abb74a Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Sun, 1 Oct 2017 09:27:44 -0700 Subject: [PATCH 070/170] revert code --- paddle/operators/recurrent_op.cc | 41 +++++++++++++++++++ paddle/operators/recurrent_op.h | 19 +++++++++ .../v2/framework/tests/test_recurrent_op.py | 3 -- 3 files changed, 60 insertions(+), 3 deletions(-) diff --git a/paddle/operators/recurrent_op.cc b/paddle/operators/recurrent_op.cc index 80de229c33..b9fba3e135 100644 --- a/paddle/operators/recurrent_op.cc +++ b/paddle/operators/recurrent_op.cc @@ -28,6 +28,29 @@ using Variable = framework::Variable; using Tensor = framework::Tensor; using LoDTensor = framework::LoDTensor; +void RecurrentAlgorithm::InferShape(const Scope& scope) const { + auto* input0 = scope.FindVar(arg_->inlinks[0]); + PADDLE_ENFORCE_NOT_NULL(input0); + seq_len_ = input0->GetMutable()->dims()[0]; + PADDLE_ENFORCE_GT(seq_len_, 0); + + CreateScopes(scope); + auto& step_scopes = GetStepScopes(scope); + rnn::SegmentInputs(step_scopes, arg_->inlinks, seq_len_, + true /*infer_shape_mode*/); + InitMemories(step_scopes[0], true /*infer_shape_mode*/); + + for (size_t i = 0; i < seq_len_; i++) { + if (i > 0) { + rnn::LinkMemories(step_scopes, arg_->memories, i, -1, + true /*infer_shape_mode*/); + } + (*stepnet_)->InferShape(*step_scopes[i]); + } + rnn::ConcatOutputs(step_scopes, arg_->outlinks, seq_len_, + true /*infer_shape_mode*/); +} + void RecurrentAlgorithm::Run(const Scope& scope, const platform::DeviceContext& dev_ctx) const { auto step_scopes = GetStepScopes(scope); @@ -179,6 +202,24 @@ void RecurrentGradientAlgorithm::LinkBootMemoryGradients( } } +void RecurrentGradientAlgorithm::InferShape(const Scope& scope) const { + seq_len_ = + scope.FindVar(arg_->inlinks[0])->GetMutable()->dims()[0]; + auto step_scopes = GetStepScopes(scope); + rnn::SegmentInputs(step_scopes, arg_->inlinks, seq_len_, + true /*infer_shape_mode*/); + for (int step_id = seq_len_ - 1; step_id >= 0; --step_id) { + if (static_cast(step_id) != seq_len_ - 1) { + rnn::LinkMemories(step_scopes, arg_->memories, step_id, 1, + true /*infer_shape_mode*/); + } + (*stepnet_)->InferShape(*step_scopes[step_id]); + } + rnn::ConcatOutputs(step_scopes, arg_->outlinks, seq_len_, + true /*infer_shape_mode*/); + LinkBootMemoryGradients(step_scopes[0], true /*infer_shape_mode*/); +} + RecurrentGradientOp::RecurrentGradientOp( const std::string& type, const framework::VariableNameMap& inputs, const framework::VariableNameMap& outputs, diff --git a/paddle/operators/recurrent_op.h b/paddle/operators/recurrent_op.h index c6b9a5533e..18f8c53e18 100644 --- a/paddle/operators/recurrent_op.h +++ b/paddle/operators/recurrent_op.h @@ -41,6 +41,11 @@ class RecurrentAlgorithm { stepnet_ = stepnet; } + /** + * InferShape must be called before Run. + */ + void InferShape(const framework::Scope& scope) const; + protected: /* * The step scopes will be stored in the father scope as a variable. @@ -89,6 +94,11 @@ class RecurrentGradientAlgorithm { void LinkBootMemoryGradients(framework::Scope* step_scopes, bool infer_shape_mode) const; + /** + * InferShape must be called before Run. + */ + void InferShape(const framework::Scope& scope) const; + protected: inline const std::vector& GetStepScopes( const framework::Scope& scope) const { @@ -123,8 +133,13 @@ class RecurrentOp : public framework::OperatorBase { void set_stepnet(std::unique_ptr net) { stepnet_ = std::move(net); } + const OperatorBase& stepnet() const { return *stepnet_; } + void InferShape(const framework::Scope& scope) const { + alg_.InferShape(scope); + } + static const rnn::ArgumentName kArgName; private: @@ -147,6 +162,10 @@ class RecurrentGradientOp : public framework::OperatorBase { PADDLE_THROW("Not Implemented"); } + void InferShape(const framework::Scope& scope) const { + alg_.InferShape(scope); + } + void Run(const framework::Scope& scope, const platform::DeviceContext& dev_ctx) const override { alg_.Run(scope, dev_ctx); diff --git a/python/paddle/v2/framework/tests/test_recurrent_op.py b/python/paddle/v2/framework/tests/test_recurrent_op.py index 92161ae5dd..6b9e7a88ce 100644 --- a/python/paddle/v2/framework/tests/test_recurrent_op.py +++ b/python/paddle/v2/framework/tests/test_recurrent_op.py @@ -197,7 +197,4 @@ class RecurrentGradientOpTest(unittest.TestCase): if __name__ == '__main__': - exit( - 0 - ) # FIXME(yuyang18): InferShape has been removed, this unittest may error unittest.main() From d25766e571f0d08c3730c4a36090772c9342fdb7 Mon Sep 17 00:00:00 2001 From: kexinzhao <19hskevin87@gmail.com> Date: Sun, 1 Oct 2017 09:36:02 -0700 Subject: [PATCH 071/170] Fix typo in new_op_cn.md --- doc/howto/dev/new_op_cn.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/howto/dev/new_op_cn.md b/doc/howto/dev/new_op_cn.md index 264b998f50..9c5c90ca03 100644 --- a/doc/howto/dev/new_op_cn.md +++ b/doc/howto/dev/new_op_cn.md @@ -206,7 +206,7 @@ MulOp(const std::string &type, const framework::VariableNameMap &inputs, - `REGISTER_OP` ： 注册`ops::MulOp`类，类型名为`mul`，该类的`ProtoMaker`为`ops::MulOpMaker`，注册`ops::MulOpGrad`，类型名为`mul_grad`。 - `REGISTER_OP_WITHOUT_GRADIENT` ： 用于注册没有反向的Op。 - - `REGISTER_OP_CPU_KERNEL` ：注册`ops::MulKernel`类，并特化模板参数为`paddle::platform::CPUPlace`和`float`类型，同理，注册`ops::MulKernel`类。 + - `REGISTER_OP_CPU_KERNEL` ：注册`ops::MulKernel`类，并特化模板参数为`paddle::platform::CPUPlace`和`float`类型，同理，注册`ops::MulGradKernel`类。 - 在 `.cu`文件中注册GPU Kernel。 From 3851fa4d8b15d0876c40c3fadc3dd54b87b5e42b Mon Sep 17 00:00:00 2001 From: kexinzhao <19hskevin87@gmail.com> Date: Sun, 1 Oct 2017 09:40:49 -0700 Subject: [PATCH 072/170] Update new_op_en.md --- doc/howto/dev/new_op_en.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/howto/dev/new_op_en.md b/doc/howto/dev/new_op_en.md index 60681cdd71..56ed22bc34 100644 --- a/doc/howto/dev/new_op_en.md +++ b/doc/howto/dev/new_op_en.md @@ -205,7 +205,7 @@ The definition of its corresponding backward operator, if applicable, is similar - `REGISTER_OP` registers the `ops::MulOp` class, type named `mul`, its type `ProtoMaker` is `ops::MulOpMaker`, registering `ops::MulOpGrad` as `mul_grad`. - `REGISTER_OP_WITHOUT_GRADIENT` registers an operator without gradient. - - `REGISTER_OP_CPU_KERNEL` registers `ops::MulKernel` class and specialized template types `paddle::platform::CPUPlace` and `float`, which also registers `ops::MulKernel`. + - `REGISTER_OP_CPU_KERNEL` registers `ops::MulKernel` class and specialized template types `paddle::platform::CPUPlace` and `float`, which also registers `ops::MulGradKernel`. - Registering GPU Kernel in `.cu` files From 5423cb3e57949fc2885e39016422bf92b70b5260 Mon Sep 17 00:00:00 2001 From: dongzhihong Date: Sun, 1 Oct 2017 09:54:08 -0700 Subject: [PATCH 073/170] format --- paddle/framework/block_desc.h | 6 +++--- paddle/framework/op_info.h | 8 +++----- paddle/framework/program_desc.h | 6 +++--- paddle/framework/scope.h | 8 +++----- paddle/platform/macros.h | 10 ++++++---- 5 files changed, 18 insertions(+), 20 deletions(-) diff --git a/paddle/framework/block_desc.h b/paddle/framework/block_desc.h index 1a1135bab4..59513ede33 100644 --- a/paddle/framework/block_desc.h +++ b/paddle/framework/block_desc.h @@ -19,6 +19,7 @@ limitations under the License. */ #include #include "paddle/framework/op_desc.h" #include "paddle/framework/var_desc.h" +#include "paddle/platform/macros.h" namespace paddle { namespace framework { @@ -34,9 +35,6 @@ class BlockDescBind { BlockDescBind(ProgramDescBind *prog, BlockDesc *desc) : prog_(prog), desc_(desc), need_update_(false) {} - BlockDescBind(const BlockDescBind &o) = delete; - BlockDescBind &operator=(const BlockDescBind &o) = delete; - int32_t ID() const { return desc_->idx(); } int32_t Parent() const { return desc_->parent_idx(); } @@ -66,6 +64,8 @@ class BlockDescBind { std::deque> ops_; std::unordered_map> vars_; + + DISABLE_COPY_AND_ASSIGN(BlockDescBind); }; } // namespace framework } // namespace paddle diff --git a/paddle/framework/op_info.h b/paddle/framework/op_info.h index 6d1ee4dece..5df3093318 100644 --- a/paddle/framework/op_info.h +++ b/paddle/framework/op_info.h @@ -19,6 +19,7 @@ #include #include "paddle/framework/attribute.h" #include "paddle/framework/op_desc.h" +#include "paddle/platform/macros.h" namespace paddle { namespace framework { @@ -72,11 +73,6 @@ class OpInfoMap { public: static OpInfoMap& Instance(); - OpInfoMap(const OpInfoMap& o) = delete; - OpInfoMap(OpInfoMap&& o) = delete; - OpInfoMap& operator=(const OpInfoMap& o) = delete; - OpInfoMap& operator=(OpInfoMap&& o) = delete; - bool Has(const std::string& op_type) const { return map_.find(op_type) != map_.end(); } @@ -112,6 +108,8 @@ class OpInfoMap { private: OpInfoMap() = default; std::unordered_map map_; + + DISABLE_COPY_AND_ASSIGN(OpInfoMap); }; } // namespace framework diff --git a/paddle/framework/program_desc.h b/paddle/framework/program_desc.h index 06ffcd4b15..9b34a06aef 100644 --- a/paddle/framework/program_desc.h +++ b/paddle/framework/program_desc.h @@ -16,6 +16,7 @@ limitations under the License. */ #include #include "paddle/framework/framework.pb.h" +#include "paddle/platform/macros.h" namespace paddle { namespace framework { @@ -26,9 +27,6 @@ class ProgramDescBind { public: static ProgramDescBind &Instance(ProgramDesc *prog); - ProgramDescBind(const ProgramDescBind &o) = delete; - ProgramDescBind &operator=(const ProgramDescBind &o) = delete; - BlockDescBind *AppendBlock(const BlockDescBind &parent); BlockDescBind *Block(size_t idx) { return blocks_[idx].get(); } @@ -46,6 +44,8 @@ class ProgramDescBind { ProgramDesc *prog_; std::vector> blocks_; + + DISABLE_COPY_AND_ASSIGN(ProgramDescBind); }; } // namespace framework } // namespace paddle diff --git a/paddle/framework/scope.h b/paddle/framework/scope.h index c93b03e481..7047f0d55e 100644 --- a/paddle/framework/scope.h +++ b/paddle/framework/scope.h @@ -19,6 +19,7 @@ limitations under the License. */ #include #include "paddle/framework/variable.h" +#include "paddle/platform/macros.h" namespace paddle { namespace framework { @@ -38,11 +39,6 @@ class Scope { Scope() {} ~Scope(); - // Disable Copy, Assign, Move. - Scope(const Scope& other) = delete; - Scope& operator=(const Scope& other) = delete; - Scope(Scope&& other) = delete; - /// Create a sub-scope. Returns a reference other than a pointer so /// to prevent from manual deletion. /// Mark it to const because that new kid scope cannot change parent scope. @@ -73,6 +69,8 @@ class Scope { std::unordered_map vars_; mutable std::list kids_; Scope const* parent_{nullptr}; + + DISABLE_COPY_AND_ASSIGN(Scope); }; } // namespace framework diff --git a/paddle/platform/macros.h b/paddle/platform/macros.h index 4a04a38c0c..feae7bdd77 100644 --- a/paddle/platform/macros.h +++ b/paddle/platform/macros.h @@ -16,8 +16,10 @@ limitations under the License. */ // Disable the copy and assignment operator for a class. #ifndef DISABLE_COPY_AND_ASSIGN -#define DISABLE_COPY_AND_ASSIGN(classname) \ - private: \ - classname(const classname&) = delete; \ - classname& operator=(const classname&) = delete +#define DISABLE_COPY_AND_ASSIGN(classname) \ + private: \ + classname(const classname&) = delete; \ + classname(const classname&&) = delete; \ + classname& operator=(const classname&) = delete; \ + classname& operator=(const classname&&) = delete #endif From 8069a3cf0a0285febe1e4785a56236aac8987e64 Mon Sep 17 00:00:00 2001 From: kexinzhao <19hskevin87@gmail.com> Date: Sun, 1 Oct 2017 09:36:02 -0700 Subject: [PATCH 074/170] Fix typo in new_op_cn.md --- doc/howto/dev/new_op_cn.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/howto/dev/new_op_cn.md b/doc/howto/dev/new_op_cn.md index 9d3d02ffc3..c823d7e9fc 100644 --- a/doc/howto/dev/new_op_cn.md +++ b/doc/howto/dev/new_op_cn.md @@ -206,7 +206,7 @@ MulOp(const std::string &type, const framework::VariableNameMap &inputs, - `REGISTER_OP` ： 注册`ops::MulOp`类，类型名为`mul`，该类的`ProtoMaker`为`ops::MulOpMaker`，注册`ops::MulOpGrad`，类型名为`mul_grad`。 - `REGISTER_OP_WITHOUT_GRADIENT` ： 用于注册没有反向的Op。 - - `REGISTER_OP_CPU_KERNEL` ：注册`ops::MulKernel`类，并特化模板参数为`paddle::platform::CPUPlace`和`float`类型，同理，注册`ops::MulKernel`类。 + - `REGISTER_OP_CPU_KERNEL` ：注册`ops::MulKernel`类，并特化模板参数为`paddle::platform::CPUPlace`和`float`类型，同理，注册`ops::MulGradKernel`类。 - 在 `.cu`文件中注册GPU Kernel。 From 646cfbfe0426973b260239575aade9470adb916c Mon Sep 17 00:00:00 2001 From: kexinzhao <19hskevin87@gmail.com> Date: Sun, 1 Oct 2017 09:40:49 -0700 Subject: [PATCH 075/170] Update new_op_en.md --- doc/howto/dev/new_op_en.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/howto/dev/new_op_en.md b/doc/howto/dev/new_op_en.md index 57ff7caad1..1e88e1f5b4 100644 --- a/doc/howto/dev/new_op_en.md +++ b/doc/howto/dev/new_op_en.md @@ -205,7 +205,7 @@ The definition of its corresponding backward operator, if applicable, is similar - `REGISTER_OP` registers the `ops::MulOp` class, type named `mul`, its type `ProtoMaker` is `ops::MulOpMaker`, registering `ops::MulOpGrad` as `mul_grad`. - `REGISTER_OP_WITHOUT_GRADIENT` registers an operator without gradient. - - `REGISTER_OP_CPU_KERNEL` registers `ops::MulKernel` class and specialized template types `paddle::platform::CPUPlace` and `float`, which also registers `ops::MulKernel`. + - `REGISTER_OP_CPU_KERNEL` registers `ops::MulKernel` class and specialized template types `paddle::platform::CPUPlace` and `float`, which also registers `ops::MulGradKernel`. - Registering GPU Kernel in `.cu` files From 2296d81cf9560437b368354229b7ceb22b67d234 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Mon, 2 Oct 2017 11:39:10 -0700 Subject: [PATCH 076/170] Use `type_defs.h` to resolve cyclic dependencies --- paddle/framework/attribute.h | 10 +--------- paddle/framework/op_desc.h | 2 +- paddle/framework/op_info.h | 7 +------ paddle/framework/type_defs.h | 38 ++++++++++++++++++++++++++++++++++++ 4 files changed, 41 insertions(+), 16 deletions(-) create mode 100644 paddle/framework/type_defs.h diff --git a/paddle/framework/attribute.h b/paddle/framework/attribute.h index c7559cefb6..d13530e340 100644 --- a/paddle/framework/attribute.h +++ b/paddle/framework/attribute.h @@ -21,20 +21,12 @@ limitations under the License. */ #include #include "paddle/framework/framework.pb.h" +#include "paddle/framework/type_defs.h" #include "paddle/platform/enforce.h" -#include "paddle/platform/variant.h" namespace paddle { namespace framework { -// The order should be as same as framework.proto -typedef boost::variant, - std::vector, std::vector, bool, - std::vector, BlockDesc*> - Attribute; - -typedef std::unordered_map AttributeMap; - ProgramDesc& GetProgramDesc(); template diff --git a/paddle/framework/op_desc.h b/paddle/framework/op_desc.h index e03b4d067f..0af4169715 100644 --- a/paddle/framework/op_desc.h +++ b/paddle/framework/op_desc.h @@ -17,7 +17,7 @@ limitations under the License. */ #include #include #include "paddle/framework/attribute.h" -#include "paddle/framework/op_info.h" +#include "paddle/framework/type_defs.h" #include "paddle/framework/var_desc.h" namespace paddle { diff --git a/paddle/framework/op_info.h b/paddle/framework/op_info.h index 6d1ee4dece..470336d367 100644 --- a/paddle/framework/op_info.h +++ b/paddle/framework/op_info.h @@ -19,15 +19,10 @@ #include #include "paddle/framework/attribute.h" #include "paddle/framework/op_desc.h" +#include "paddle/framework/type_defs.h" namespace paddle { namespace framework { -class OperatorBase; -using VariableNameMap = std::map>; - -using OpCreator = std::function; class GradOpDescMakerBase { public: diff --git a/paddle/framework/type_defs.h b/paddle/framework/type_defs.h new file mode 100644 index 0000000000..dec5066f1e --- /dev/null +++ b/paddle/framework/type_defs.h @@ -0,0 +1,38 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once +#include +#include +#include "paddle/platform/variant.h" + +namespace paddle { +namespace framework { +class OperatorBase; +using VariableNameMap = std::map>; + +// The order should be as same as framework.proto +using Attribute = + boost::variant, + std::vector, std::vector, bool, + std::vector, BlockDesc*>; + +using AttributeMap = std::unordered_map; + +using OpCreator = std::function; + +} // namespace framework +} // namespace paddle From 32f5c9dd934e7de15a93a8145bf6ee4499b3bc7d Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Mon, 2 Oct 2017 11:51:24 -0700 Subject: [PATCH 077/170] recurrent_op pass the unit test --- paddle/operators/recurrent_op.cc | 87 +++++-------------- paddle/operators/recurrent_op.h | 23 +---- paddle/operators/rnn/recurrent_op_utils.cc | 55 +++++------- paddle/operators/rnn/recurrent_op_utils.h | 6 +- paddle/operators/sum_op.cc | 5 +- .../v2/framework/tests/test_recurrent_op.py | 26 +++--- 6 files changed, 66 insertions(+), 136 deletions(-) diff --git a/paddle/operators/recurrent_op.cc b/paddle/operators/recurrent_op.cc index b9fba3e135..016e2043fd 100644 --- a/paddle/operators/recurrent_op.cc +++ b/paddle/operators/recurrent_op.cc @@ -28,7 +28,8 @@ using Variable = framework::Variable; using Tensor = framework::Tensor; using LoDTensor = framework::LoDTensor; -void RecurrentAlgorithm::InferShape(const Scope& scope) const { +void RecurrentAlgorithm::Run(const Scope& scope, + const platform::DeviceContext& dev_ctx) const { auto* input0 = scope.FindVar(arg_->inlinks[0]); PADDLE_ENFORCE_NOT_NULL(input0); seq_len_ = input0->GetMutable()->dims()[0]; @@ -36,38 +37,16 @@ void RecurrentAlgorithm::InferShape(const Scope& scope) const { CreateScopes(scope); auto& step_scopes = GetStepScopes(scope); - rnn::SegmentInputs(step_scopes, arg_->inlinks, seq_len_, - true /*infer_shape_mode*/); - InitMemories(step_scopes[0], true /*infer_shape_mode*/); + rnn::SegmentInputs(step_scopes, arg_->inlinks, seq_len_); + InitMemories(step_scopes[0]); for (size_t i = 0; i < seq_len_; i++) { if (i > 0) { - rnn::LinkMemories(step_scopes, arg_->memories, i, -1, - true /*infer_shape_mode*/); - } - (*stepnet_)->InferShape(*step_scopes[i]); - } - rnn::ConcatOutputs(step_scopes, arg_->outlinks, seq_len_, - true /*infer_shape_mode*/); -} - -void RecurrentAlgorithm::Run(const Scope& scope, - const platform::DeviceContext& dev_ctx) const { - auto step_scopes = GetStepScopes(scope); - rnn::SegmentInputs(step_scopes, arg_->inlinks, seq_len_, - false /*infer_shape_mode*/); - InitMemories(step_scopes[0], false /*infer_shape_mode*/); - - for (size_t step_id = 0; step_id < seq_len_; step_id++) { - // create output alias variables - if (step_id > 0) { - rnn::LinkMemories(step_scopes, arg_->memories, step_id, -1, - false /*infer_shape_mode*/); + rnn::LinkMemories(step_scopes, arg_->memories, i, -1); } - (*stepnet_)->Run(*step_scopes[step_id], dev_ctx); + (*stepnet_)->Run(*step_scopes[i], dev_ctx); } - rnn::ConcatOutputs(step_scopes, arg_->outlinks, seq_len_, - false /*infer_shape_mode*/); + rnn::ConcatOutputs(step_scopes, arg_->outlinks, seq_len_); } void RecurrentAlgorithm::CreateScopes(const Scope& scope) const { @@ -105,8 +84,7 @@ void RecurrentAlgorithm::CreateScopes(const Scope& scope) const { } } -void RecurrentAlgorithm::InitMemories(Scope* step_scope, - bool infer_shape_mode) const { +void RecurrentAlgorithm::InitMemories(Scope* step_scope) const { for (auto& attr : arg_->memories) { auto* pre_mem = step_scope->NewVar(attr.pre_var)->GetMutable(); PADDLE_ENFORCE(step_scope->FindVar(attr.boot_var) != nullptr, @@ -114,12 +92,9 @@ void RecurrentAlgorithm::InitMemories(Scope* step_scope, attr.boot_var); auto* boot_mem = step_scope->FindVar(attr.boot_var)->GetMutable(); - if (infer_shape_mode) { - pre_mem->Resize(boot_mem->dims()); - PADDLE_ENFORCE_EQ(pre_mem->dims().size(), 2); - } else { - pre_mem->ShareDataWith(*boot_mem); - } + pre_mem->Resize(boot_mem->dims()); + PADDLE_ENFORCE_EQ(pre_mem->dims().size(), 2); + pre_mem->ShareDataWith(*boot_mem); } } @@ -169,23 +144,22 @@ class RecurrentAlgorithmProtoAndCheckerMaker void RecurrentGradientAlgorithm::Run( const Scope& scope, const platform::DeviceContext& dev_ctx) const { + seq_len_ = + scope.FindVar(arg_->inlinks[0])->GetMutable()->dims()[0]; auto step_scopes = GetStepScopes(scope); - rnn::SegmentInputs(step_scopes, arg_->inlinks, seq_len_, - false /*infer_shape_mode*/); + rnn::SegmentInputs(step_scopes, arg_->inlinks, seq_len_); for (int step_id = seq_len_ - 1; step_id >= 0; --step_id) { if (static_cast(step_id) != seq_len_ - 1) { - rnn::LinkMemories(step_scopes, arg_->memories, step_id, 1, - false /*infer_shape_mode*/); + rnn::LinkMemories(step_scopes, arg_->memories, step_id, 1); } (*stepnet_)->Run(*step_scopes[step_id], dev_ctx); } - LinkBootMemoryGradients(step_scopes[0], false); - rnn::ConcatOutputs(step_scopes, arg_->outlinks, seq_len_, - false /*infer_shape_mode*/); + rnn::ConcatOutputs(step_scopes, arg_->outlinks, seq_len_); + LinkBootMemoryGradients(step_scopes[0]); } void RecurrentGradientAlgorithm::LinkBootMemoryGradients( - Scope* step_scope, bool infer_shape_mode) const { + Scope* step_scope) const { for (auto& attr : arg_->memories) { PADDLE_ENFORCE(step_scope->FindVar(attr.var) != nullptr, "memory variable [%s] does not exists", attr.var); @@ -194,30 +168,9 @@ void RecurrentGradientAlgorithm::LinkBootMemoryGradients( auto* mem_grad = step_scope->NewVar(attr.var)->GetMutable(); auto* boot_mem_grad = step_scope->NewVar(attr.boot_var)->GetMutable(); - if (infer_shape_mode) { - boot_mem_grad->Resize(mem_grad->dims()); - } else { - boot_mem_grad->ShareDataWith(*mem_grad); - } - } -} - -void RecurrentGradientAlgorithm::InferShape(const Scope& scope) const { - seq_len_ = - scope.FindVar(arg_->inlinks[0])->GetMutable()->dims()[0]; - auto step_scopes = GetStepScopes(scope); - rnn::SegmentInputs(step_scopes, arg_->inlinks, seq_len_, - true /*infer_shape_mode*/); - for (int step_id = seq_len_ - 1; step_id >= 0; --step_id) { - if (static_cast(step_id) != seq_len_ - 1) { - rnn::LinkMemories(step_scopes, arg_->memories, step_id, 1, - true /*infer_shape_mode*/); - } - (*stepnet_)->InferShape(*step_scopes[step_id]); + boot_mem_grad->Resize(mem_grad->dims()); + boot_mem_grad->ShareDataWith(*mem_grad); } - rnn::ConcatOutputs(step_scopes, arg_->outlinks, seq_len_, - true /*infer_shape_mode*/); - LinkBootMemoryGradients(step_scopes[0], true /*infer_shape_mode*/); } RecurrentGradientOp::RecurrentGradientOp( diff --git a/paddle/operators/recurrent_op.h b/paddle/operators/recurrent_op.h index 18f8c53e18..752025e42c 100644 --- a/paddle/operators/recurrent_op.h +++ b/paddle/operators/recurrent_op.h @@ -41,11 +41,6 @@ class RecurrentAlgorithm { stepnet_ = stepnet; } - /** - * InferShape must be called before Run. - */ - void InferShape(const framework::Scope& scope) const; - protected: /* * The step scopes will be stored in the father scope as a variable. @@ -61,7 +56,7 @@ class RecurrentAlgorithm { ->GetMutable>(); } - void InitMemories(framework::Scope* step_scopes, bool infer_shape_mode) const; + void InitMemories(framework::Scope* step_scopes) const; private: std::unique_ptr* stepnet_; @@ -91,13 +86,7 @@ class RecurrentGradientAlgorithm { void Run(const framework::Scope& scope, const platform::DeviceContext& dev_ctx) const; - void LinkBootMemoryGradients(framework::Scope* step_scopes, - bool infer_shape_mode) const; - - /** - * InferShape must be called before Run. - */ - void InferShape(const framework::Scope& scope) const; + void LinkBootMemoryGradients(framework::Scope* step_scopes) const; protected: inline const std::vector& GetStepScopes( @@ -136,10 +125,6 @@ class RecurrentOp : public framework::OperatorBase { const OperatorBase& stepnet() const { return *stepnet_; } - void InferShape(const framework::Scope& scope) const { - alg_.InferShape(scope); - } - static const rnn::ArgumentName kArgName; private: @@ -162,10 +147,6 @@ class RecurrentGradientOp : public framework::OperatorBase { PADDLE_THROW("Not Implemented"); } - void InferShape(const framework::Scope& scope) const { - alg_.InferShape(scope); - } - void Run(const framework::Scope& scope, const platform::DeviceContext& dev_ctx) const override { alg_.Run(scope, dev_ctx); diff --git a/paddle/operators/rnn/recurrent_op_utils.cc b/paddle/operators/rnn/recurrent_op_utils.cc index a767009d23..a02994f99d 100644 --- a/paddle/operators/rnn/recurrent_op_utils.cc +++ b/paddle/operators/rnn/recurrent_op_utils.cc @@ -25,7 +25,7 @@ using LoDTensor = framework::LoDTensor; void SegmentInputs(const std::vector& step_scopes, const std::vector& inlinks, - const size_t seq_len, bool infer_shape_mode) { + const size_t seq_len) { PADDLE_ENFORCE(!inlinks.empty(), "no in links are provided."); for (size_t i = 0; i < inlinks.size(); ++i) { // global inputs @@ -41,11 +41,9 @@ void SegmentInputs(const std::vector& step_scopes, for (size_t j = 0; j < seq_len; j++) { Tensor* step_input = step_scopes[j]->NewVar(inlinks[i])->GetMutable(); - if (!infer_shape_mode) { - // The input of operators of each step is Tensor here. - // Maybe need to modify Slice function. - *step_input = input->Slice(j, j + 1); - } + // The input of operators of each step is Tensor here. + // Maybe need to modify Slice function. + *step_input = input->Slice(j, j + 1); step_input->Resize(step_dims); } } @@ -53,39 +51,35 @@ void SegmentInputs(const std::vector& step_scopes, void ConcatOutputs(const std::vector& step_scopes, const std::vector& outlinks, - const size_t seq_len, bool infer_shape_mode) { + const size_t seq_len) { for (size_t i = 0; i < outlinks.size(); i++) { auto output_var = step_scopes[0]->parent().FindVar(outlinks[i]); PADDLE_ENFORCE_NOT_NULL(output_var, "output link [%s] is not in scope.", outlinks[i]); LoDTensor* output = output_var->GetMutable(); - if (infer_shape_mode) { - auto step_scope_var = step_scopes[0]->FindVar(outlinks[i]); - PADDLE_ENFORCE_NOT_NULL(step_scope_var, "%s not in scope", outlinks[i]); - f::DDim step_dims = - step_scope_var->template GetMutable()->dims(); - std::vector dims_vec = vectorize(step_dims); - dims_vec.insert(dims_vec.begin(), seq_len); - output->Resize(f::make_ddim(dims_vec)); - } else { - output->mutable_data(platform::CPUPlace()); - for (size_t j = 0; j < seq_len; j++) { - LoDTensor* step_output = - step_scopes[j]->FindVar(outlinks[i])->GetMutable(); - // TODO(luotao02) data type and platform::DeviceContext() should set - // correctly - (output->Slice(j, j + 1)) - .CopyFrom(*step_output, platform::CPUPlace()); - } + auto step_scope_var = step_scopes[0]->FindVar(outlinks[i]); + PADDLE_ENFORCE_NOT_NULL(step_scope_var, "%s not in scope", outlinks[i]); + f::DDim step_dims = + step_scope_var->template GetMutable()->dims(); + std::vector dims_vec = vectorize(step_dims); + dims_vec.insert(dims_vec.begin(), seq_len); + output->Resize(f::make_ddim(dims_vec)); + output->mutable_data(platform::CPUPlace()); + for (size_t j = 0; j < seq_len; j++) { + LoDTensor* step_output = + step_scopes[j]->FindVar(outlinks[i])->GetMutable(); + // TODO(luotao02) data type and platform::DeviceContext() should set + // correctly + (output->Slice(j, j + 1)) + .CopyFrom(*step_output, platform::CPUPlace()); } } } void LinkMemories(const std::vector& scopes, const std::vector& memories, - const size_t step_id, const int offset, - bool infer_shape_mode) { + const size_t step_id, const int offset) { PADDLE_ENFORCE_LT(step_id, scopes.size(), "step [%d] is out of range of step scopes' size [%d]", step_id, scopes.size()); @@ -100,11 +94,8 @@ void LinkMemories(const std::vector& scopes, for (auto& attr : memories) { auto mem = scope->FindVar(attr.pre_var)->GetMutable(); auto linked_mem = linked_scope->FindVar(attr.var)->GetMutable(); - if (infer_shape_mode) { - mem->Resize(linked_mem->dims()); - } else { - mem->ShareDataWith(*linked_mem); - } + mem->Resize(linked_mem->dims()); + mem->ShareDataWith(*linked_mem); } } diff --git a/paddle/operators/rnn/recurrent_op_utils.h b/paddle/operators/rnn/recurrent_op_utils.h index 9c777f1e90..fd17b9b889 100644 --- a/paddle/operators/rnn/recurrent_op_utils.h +++ b/paddle/operators/rnn/recurrent_op_utils.h @@ -64,18 +64,18 @@ struct ArgumentName { */ void SegmentInputs(const std::vector& step_scopes, const std::vector& inlinks, - const size_t seq_len, bool infer_shape_mode); + const size_t seq_len); /** * Process outputs of step nets and merge to variables. */ void ConcatOutputs(const std::vector& step_scopes, const std::vector& outlinks, - const size_t seq_len, bool infer_shape_mode); + const size_t seq_len); void LinkMemories(const std::vector& step_scopes, const std::vector& memories, const size_t step_id, - const int offset, bool infer_shape_mode); + const int offset); void InitArgument(const ArgumentName& name, Argument* arg, const framework::OperatorBase& op, bool is_grad = false); diff --git a/paddle/operators/sum_op.cc b/paddle/operators/sum_op.cc index 5d76313aeb..c54843faa6 100644 --- a/paddle/operators/sum_op.cc +++ b/paddle/operators/sum_op.cc @@ -22,14 +22,15 @@ class SumOp : public framework::OperatorWithKernel { protected: void InferShape(framework::InferShapeContextBase* ctx) const override { + PADDLE_ENFORCE(ctx->HasInputs("X"), "Inputs(X) should not be null"); auto x_dims = ctx->GetInputsDim("X"); - PADDLE_ENFORCE(!x_dims.empty(), "Input(X) of SumOp should not be null."); PADDLE_ENFORCE(ctx->HasOutput("Out"), "Output(Out) of SumOp should not be null."); - auto in_dim = x_dims[0]; size_t N = x_dims.size(); PADDLE_ENFORCE_GT(N, 1, "Input tensors count should > 1."); + + auto in_dim = x_dims[0]; for (size_t i = 1; i < N; i++) { auto dim = x_dims[i]; PADDLE_ENFORCE(in_dim == dim, "Input tensors must have same shape"); diff --git a/python/paddle/v2/framework/tests/test_recurrent_op.py b/python/paddle/v2/framework/tests/test_recurrent_op.py index 6b9e7a88ce..1f114432c0 100644 --- a/python/paddle/v2/framework/tests/test_recurrent_op.py +++ b/python/paddle/v2/framework/tests/test_recurrent_op.py @@ -16,14 +16,17 @@ class PySimpleRNN(object): ''' def __init__(self, input_dim=30, batch_size=50, weight_dim=15, sent_len=11): - self.x = np.random.normal(size=(sent_len, batch_size, input_dim)) - self.W = np.random.normal(size=(input_dim, input_dim)) - self.U = np.random.normal(size=(input_dim, input_dim)) - self.h_boot = np.random.normal(size=(batch_size, input_dim)) + self.x = np.random.normal(size=(sent_len, batch_size, + input_dim)).astype("float32") + self.W = np.random.normal(size=(input_dim, input_dim)).astype("float32") + self.U = np.random.normal(size=(input_dim, input_dim)).astype("float32") + self.h_boot = np.random.normal(size=(batch_size, + input_dim)).astype("float32") # memories self.mems = [ - np.zeros(shape=(batch_size, input_dim)) for i in range(sent_len) + np.zeros(shape=(batch_size, input_dim)).astype("float32") + for i in range(sent_len) ] def forward(self): @@ -36,7 +39,7 @@ class PySimpleRNN(object): return [self.x[i] for i in range(self.x.shape[0])] def concat_outputs(self): - return np.array(self.mems) + return np.array(self.mems).astype("float32") def step(self, step_id, x): ''' @@ -47,8 +50,8 @@ class PySimpleRNN(object): pre_mem = self.mems[step_id - 1] else: pre_mem = self.h_boot - xW = np.matmul(x, self.W) - hU = np.matmul(pre_mem, self.U) + xW = np.matmul(x, self.W).astype("float32") + hU = np.matmul(pre_mem, self.U).astype("float32") sum = xW + hU self.mems[step_id] = py_sigmoid(sum) @@ -102,7 +105,8 @@ class RecurrentOpTest(unittest.TestCase): self.create_step_net() ctx = core.DeviceContext.create(core.CPUPlace()) self.rnnop.run(self.scope, ctx) - return np.array(self.scope.find_var("h@mem").get_tensor()) + return np.array(self.scope.find_var("h@mem").get_tensor()).astype( + "float32") def create_global_variables(self): # create inlink @@ -142,7 +146,7 @@ class RecurrentOpTest(unittest.TestCase): stepnet = core.Net.create() x_fc_op = Operator("mul", X="x", Y="W", Out="Wx") h_fc_op = Operator("mul", X="h@pre", Y="U", Out="Uh") - sum_op = Operator("add", X="Wx", Y="Uh", Out="sum") + sum_op = Operator("sum", X=["Wx", "Uh"], Out="sum") sig_op = Operator("sigmoid", X="sum", Y="h@mem") for op in [x_fc_op, h_fc_op, sum_op, sig_op]: @@ -179,7 +183,7 @@ class RecurrentGradientOpTest(unittest.TestCase): stepnet = core.Net.create() x_fc_op = Operator("mul", X="x@alias", Y="W", Out="Wx") h_fc_op = Operator("mul", X="h@pre", Y="U", Out="Uh") - sum_op = Operator("add", X="Wx", Y="Uh", Out="sum") + sum_op = Operator("sum", X=["Wx", "Uh"], Out="sum") sig_op = Operator("sigmoid", X="sum", Y="h@alias") for op in [x_fc_op, h_fc_op, sum_op, sig_op]: From 3723304da953cce7aa88d1fdbd684bff91412dae Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Mon, 2 Oct 2017 11:56:14 -0700 Subject: [PATCH 078/170] Add missing ctor --- paddle/framework/grad_op_desc_maker.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/paddle/framework/grad_op_desc_maker.h b/paddle/framework/grad_op_desc_maker.h index cb4d160bd0..b4b6d54bf3 100644 --- a/paddle/framework/grad_op_desc_maker.h +++ b/paddle/framework/grad_op_desc_maker.h @@ -79,6 +79,8 @@ class GradOpDescMakerBase { class SingleGradOpDescMaker : public GradOpDescMakerBase { public: + using GradOpDescMakerBase::GradOpDescMakerBase; + std::vector operator()() const { return {this->Apply()}; } protected: @@ -86,6 +88,9 @@ class SingleGradOpDescMaker : public GradOpDescMakerBase { }; class DefaultGradOpDescMaker : public SingleGradOpDescMaker { + public: + using SingleGradOpDescMaker::SingleGradOpDescMaker; + protected: virtual OpDescBind Apply() const { OpDescBind grad; From c705f065ba403606d39bc972d85f7eba1920f029 Mon Sep 17 00:00:00 2001 From: Yan Chunwei Date: Mon, 2 Oct 2017 16:14:48 -0400 Subject: [PATCH 079/170] add TensorArray (#4459) * add tensor array * update * set type --- paddle/framework/CMakeLists.txt | 3 + paddle/framework/tensor_array.cc | 283 ++++++++++++++++++++++++++ paddle/framework/tensor_array.h | 118 +++++++++++ paddle/framework/tensor_array_test.cc | 130 ++++++++++++ 4 files changed, 534 insertions(+) create mode 100644 paddle/framework/tensor_array.cc create mode 100644 paddle/framework/tensor_array.h create mode 100644 paddle/framework/tensor_array_test.cc diff --git a/paddle/framework/CMakeLists.txt b/paddle/framework/CMakeLists.txt index 9140854a96..5d394132b7 100644 --- a/paddle/framework/CMakeLists.txt +++ b/paddle/framework/CMakeLists.txt @@ -43,3 +43,6 @@ add_custom_command(TARGET framework_py_proto POST_BUILD cc_library(backward SRCS backward.cc DEPS net_op) cc_test(backward_test SRCS backward_test.cc DEPS backward recurrent_op device_context) + +cc_library(tensor_array SRCS tensor_array.cc DEPS lod_tensor) +cc_test(tensor_array_test SRCS tensor_array_test.cc DEPS tensor_array place) diff --git a/paddle/framework/tensor_array.cc b/paddle/framework/tensor_array.cc new file mode 100644 index 0000000000..d54714c66c --- /dev/null +++ b/paddle/framework/tensor_array.cc @@ -0,0 +1,283 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + + + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/framework/tensor_array.h" + +#include +#include +#include + +namespace paddle { +namespace framework { + +namespace detail { + +/* + * Offer an iterator over the length-sorted lod-tensor's top level. The top + * level of a lod-tensor stores batch-size of sequences, each top-level sequence + * may contains several lower-level sequences, sort top-level lod by the numbers + * of lower-level sequences in descending order, so that during RNN's running, + * the batch-size will keep decreasing, the short sentences will end at the tail + * of each batch. + * + * Let's take a simple lod-tensor for example + * + * |(0) |(1) top-level has two instances + * ||| ||||| lower-level + * + * sort by lower-level's length + * + * |(1) |(0) + * ||||| ||| + * + * when RNN runs, it get 5 batches (equals the number of elements the longest + * sequence has) + * + * ||||| + * ||| + * + * the first three batches has two elements, the last two elements just has 1 + * element each. + */ +struct DynamicBatchUnpacker { + using value_type = float; + + DynamicBatchUnpacker(const LoDTensor& source, size_t level, + bool descend = true) + : source(&source), level(level) { + BuildLengthSortedMeta(descend); + } + + LoDTensor GetBatch(size_t index); + + std::vector meta; + + LoDTensor const* source; + size_t level; + + protected: + void BuildLengthSortedMeta(bool descend); +}; + +LoDTensor PackDynamicBatch(const std::vector& source, + const std::vector& meta, const LoD& lod, + size_t level); + +} // namespace detail + +const LoDTensor& TensorArray::Read(size_t index) const { + PADDLE_ENFORCE_LE(index, MAX_SIZE, "index[%d] too large", index); + if (index >= size()) { + values_.resize(index + 1); + } + return values_[index]; +} + +void TensorArray::Write(size_t index, const LoDTensor& value) { + PADDLE_ENFORCE_LE(index, MAX_SIZE, "index[%d] too large", index); + + if (index >= size()) { + values_.resize(index + 1); + } + + values_[index].Resize(value.dims()); + values_[index].mutable_data(platform::CPUPlace()); + values_[index].CopyFrom(value, platform::CPUPlace()); +} + +void TensorArray::WriteShared(size_t index, const LoDTensor& value) { + PADDLE_ENFORCE_LE(index, MAX_SIZE, "index[%d] too large", index); + if (index >= size()) { + values_.resize(index + 1); + } + + values_[index].ShareDataWith(value); +} + +LoDTensor TensorArray::Pack(size_t level, const std::vector& meta, + const LoD& lod) const { + return detail::PackDynamicBatch(values_, meta, lod, level); +} + +std::vector TensorArray::Unpack(const LoDTensor& source, int level, + bool length_desend) { + detail::DynamicBatchUnpacker unpacker(source, level, + length_desend /*descend*/); + + // find max length of all the sequences + size_t max_length = 0; + for (const auto& seq : unpacker.meta) { + max_length = std::max(max_length, seq.end - seq.begin); + } + + // write batches to values + for (size_t batch_id = 0; batch_id < max_length; batch_id++) { + Write(batch_id, unpacker.GetBatch(batch_id)); + } + + return unpacker.meta; +} + +LoDTensor TensorArray::Stack() const { + LoDTensor result; + if (size() == 0) return result; + + const auto& first_dims = values_.front().dims(); + // check all the values have the same shape + // TODO(superjom) check the same dtypes + for (size_t idx = 1; idx < size(); idx++) { + const auto& value_dims = values_[idx].dims(); + PADDLE_ENFORCE_EQ(first_dims, value_dims); + } + + // copy + auto result_dims = vectorize(first_dims); + result_dims.insert(result_dims.begin(), size()); + result.Resize(make_ddim(result_dims)); + result.mutable_data(platform::CPUPlace()); + + for (size_t idx = 0; idx < size(); idx++) { + result.Slice(idx, idx + 1) + .CopyFrom(Read(idx), platform::CPUPlace()); + } + return result; +} + +void TensorArray::Unstack(const LoDTensor& source) const { + Unstack(source, false /*data_shared*/); +} + +void TensorArray::UnstackShared(const LoDTensor& source) const { + Unstack(source, true /*data_shared*/); +} + +void TensorArray::Unstack(const LoDTensor& source, bool data_shared) const { + size_t first_dim = source.dims()[0]; + DDim value_dims = slice_ddim(source.dims(), 1, source.dims().size()); + PADDLE_ENFORCE_GT(first_dim, 0, + "source should have some data to be unstacked"); + + values_.resize(first_dim); + + for (size_t elem = 0; elem < first_dim; elem++) { + // create a new value + auto& value = values_[elem]; + if (data_shared) { + // share memory + value.ShareDataWith(source.Slice(elem, elem + 1)); + } else { + // copy + value.Resize(value_dims); + value.CopyFrom(source.Slice(elem, elem + 1), + platform::CPUPlace()); + } + } +} + +size_t TensorArray::size() const { return values_.size(); } + +namespace detail { + +void DynamicBatchUnpacker::BuildLengthSortedMeta(bool descend) { + PADDLE_ENFORCE(meta.empty(), "duplicate build meta"); + // collect meta for each sequence in some level + auto lod = SliceLevels(source->lod(), level, level + 1)[0]; + + for (size_t seq_id = 0; seq_id < lod.size() - 1; seq_id++) { + DySeqMeta seq_meta({lod[seq_id], lod[seq_id + 1], seq_id}); + meta.push_back(seq_meta); + } + + PADDLE_ENFORCE_GT(meta.size(), 0, "meta is empty"); + + // sort by length + sort(meta.begin(), meta.end(), + [descend](const DySeqMeta& a, const DySeqMeta& b) { + bool a_ge_b = (a.end - a.begin) > (b.end - b.begin); + return descend ? a_ge_b : !a_ge_b; + }); +} + +LoDTensor DynamicBatchUnpacker::GetBatch(size_t index) { + PADDLE_ENFORCE(!meta.empty(), "should build meta first"); + LoDTensor result; + + // collect indice need to copy to the batch + std::vector indice; + for (size_t seq_id = 0; seq_id < meta.size(); seq_id++) { + const auto& seq_meta = meta[seq_id]; + if (index >= seq_meta.end) break; + indice.push_back(seq_meta.begin + index); + } + + PADDLE_ENFORCE(!indice.empty(), "invalid batch at %d", index); + + // copy the indice of records in LoDTensor + auto record_dims = slice_ddim(source->dims(), 1, source->dims().size()); + auto record_dims_vec = vectorize(record_dims); + record_dims_vec.insert(record_dims_vec.begin(), indice.size()); + result.Resize(make_ddim(record_dims_vec)); + result.mutable_data(platform::CPUPlace()); + + for (size_t i = 0; i < indice.size() - 1; i++) { + auto index = indice[i]; + auto target = result.Slice(i, i + 1); + auto source_ = source->Slice(index, index + 1); + target.CopyFrom(source_, platform::CPUPlace()); + } + + return result; +} + +LoDTensor PackDynamicBatch(const std::vector& source, + const std::vector& meta, const LoD& lod, + size_t level) { + PADDLE_ENFORCE(!source.empty()); + PADDLE_ENFORCE(!meta.empty()); + PADDLE_ENFORCE(!lod.empty()); + + LoDTensor result; + + // init result space + auto record_dims = slice_ddim(source[0].dims(), 1, source[0].dims().size()); + auto record_dims_vec = vectorize(record_dims); + auto height = lod[level].back(); + record_dims_vec.insert(record_dims_vec.begin(), height); + result.Resize(make_ddim(record_dims_vec)); + result.mutable_data(platform::CPUPlace()); + + for (size_t batch_id = 0; batch_id < source.size(); batch_id++) { + for (size_t seq_id = 0; seq_id < meta.size(); seq_id++) { + const auto& seq_meta = meta[seq_id]; + // source is source[batch_id][seq_id] + // target is result[index] + auto index = seq_meta.begin + batch_id; + if (index >= seq_meta.end) break; + auto source_ = source[batch_id].Slice(seq_id, seq_id + 1); + auto target = result.Slice(index, index + 1); + target.CopyFrom(source_, platform::CPUPlace()); + } + } + + result.set_lod(lod); + + return result; +} + +} // namespace detail + +} // namespace framework +} // namespace paddle diff --git a/paddle/framework/tensor_array.h b/paddle/framework/tensor_array.h new file mode 100644 index 0000000000..e76f33d2c0 --- /dev/null +++ b/paddle/framework/tensor_array.h @@ -0,0 +1,118 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once +#include + +#include "paddle/framework/lod_tensor.h" + +namespace paddle { +namespace framework { + +/* + * DyBatchSeqPosition stores indices of the basic element in tensor. It is used + * after lod-tensor's re-assembling, its info can be used to recover the order + * in original lod-tensor. + */ +struct DySeqMeta { + size_t begin; + size_t end; // not included + size_t ori_idx; +}; + +/* + * TensorArray is a C-array-like array of tensors, it is meant to be used with + * dynamic iteration primitives such as while_loop. It is used to segment inputs + * and store states in all time steps. + * + * By providing some methods similar to a C++ array, the difinition of some + * state-based dynamic models such as RNN cound be more natural and highly + * flexible. + */ +class TensorArray { + public: + using value_type = float; + + // max number of values allowed to store. + const size_t MAX_SIZE{100000}; + + /* + * Inputs: + * - value_shared: share memory between tensors. + */ + explicit TensorArray(bool values_shared = true) + : values_shared_(values_shared) {} + + /* + * Read the value at location `index` in the `TensorArray`. + */ + const LoDTensor &Read(size_t index) const; + + /* + * Write value into the index of the TensorArray. + */ + void Write(size_t index, const LoDTensor &value); + + /* + * Write value into the index of the TensorArray, with memory shared. + */ + void WriteShared(size_t index, const LoDTensor &value); + + /* + * Recover the original LoD-arranged LoDTensor with the `values`, `level` and + * `indice_map`. + */ + LoDTensor Pack(size_t level, const std::vector &meta, + const LoD &lod) const; + + /* + * Split LoDTensor in some `level` and write the generated batches to + * `values`, if set `desend`, will sort by length in descending order else in + * ascending order. + */ + std::vector Unpack(const LoDTensor &source, int level, + bool length_desend); + + /* + * Pack the values into a tensor with rank one higher than each tensor in + * values. + */ + LoDTensor Stack() const; + + /* + * Unpacks the given division of a rank-`R` tensor into rank-`(R-1)` tensors. + */ + void Unstack(const LoDTensor &source) const; + + /* + * Unpacks the given division of a rank-`R` tensor into rank-`(R-1)` tensors, + * with memory of tensors shared. + */ + void UnstackShared(const LoDTensor &source) const; + + /* + * Return the number of values. + */ + size_t size() const; + + protected: + void Unstack(const LoDTensor &source, bool data_shared) const; + + private: + mutable std::vector values_; + bool values_shared_; +}; // class TensorArray + +} // namespace framework +} // namespace paddle diff --git a/paddle/framework/tensor_array_test.cc b/paddle/framework/tensor_array_test.cc new file mode 100644 index 0000000000..d9f52509cd --- /dev/null +++ b/paddle/framework/tensor_array_test.cc @@ -0,0 +1,130 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#include "paddle/framework/tensor_array.h" + +#include + +namespace paddle { +namespace framework { + +class TensorArrayTester : public ::testing::Test { + protected: + void SetUp() override { + LoDTensor source; + source.Resize(make_ddim({batch_size, dim})); + int* data = source.mutable_data(platform::CPUPlace()); + for (int i = 0; i < 16 * 32; i++) { + data[i] = i; + } + ta.Unstack(source); + } + + TensorArray ta; + const int batch_size = 16; + const int dim = 32; +}; + +TEST_F(TensorArrayTester, Read) { + for (int i = 0; i < batch_size; i++) { + const auto& tensor = ta.Read(i); + ASSERT_EQ(tensor.dims()[0], 1); + ASSERT_EQ(tensor.dims()[1], dim); + } +} + +TEST_F(TensorArrayTester, Write) { + LoDTensor source; + source.Resize(make_ddim({1, dim})); + for (int i = 0; i < dim; i++) { + *(source.mutable_data(platform::CPUPlace()) + i) = i; + } + + ta.Write(2, source); + + const auto& tensor = ta.Read(2); + for (int i = 0; i < dim; i++) { + EXPECT_EQ(*(tensor.data() + i), *(source.data() + i)); + } +} + +TEST_F(TensorArrayTester, WriteShared) { + LoDTensor source; + source.Resize(make_ddim({1, dim})); + for (int i = 0; i < dim; i++) { + *(source.mutable_data(platform::CPUPlace()) + i) = i; + } + + ta.WriteShared(2, source); + + const auto& tensor = ta.Read(2); + for (int i = 0; i < dim; i++) { + EXPECT_EQ(*(tensor.data() + i), *(source.data() + i)); + } + + EXPECT_EQ(source.data(), tensor.data()); +} + +class TensorArrayPackTester : public ::testing::Test { + protected: + virtual void SetUp() override { + lod.push_back(std::vector{0, 2, 9, 13}); + + source.set_lod(lod); + source.Resize(make_ddim({13, 128})); + source.mutable_data(platform::CPUPlace()); + + // content of each setence: 0 1 2 3 4 + const auto& level = lod.front(); + for (size_t i = 0; i < level.size() - 1; i++) { + size_t begin = level[i]; + size_t end = level[i + 1]; + for (size_t j = begin; j < end; j++) { + auto record = source.Slice(j, j + 1); + for (int dim = 0; dim < 128; dim++) { + record.mutable_data(platform::CPUPlace())[dim] = j - begin; + } + } + } + + // unpack + meta = ta.Unpack(source, 0, true); + } + + LoD lod; + TensorArray ta; + LoDTensor source; + std::vector meta; +}; + +TEST_F(TensorArrayPackTester, Unpack) { + ASSERT_EQ(ta.size(), 7UL); + + const auto& t0 = ta.Read(0); + const auto& t1 = ta.Read(1); + + ASSERT_EQ(t0.data()[0], int(0)); + ASSERT_EQ(t1.data()[0], int(1)); +} + +TEST_F(TensorArrayPackTester, Pack) { + LoDTensor packed = ta.Pack(0, meta, lod); +} + +TEST_F(TensorArrayTester, size) { + ASSERT_EQ(ta.size(), static_cast(batch_size)); +} + +} // namespace framework +} // namespace paddle From 9b54ad18f8c6c974cff39b9e128a3ef82dd57455 Mon Sep 17 00:00:00 2001 From: caoying03 Date: Sat, 30 Sep 2017 16:06:52 +0800 Subject: [PATCH 080/170] add configuration helper for resize layer. --- doc/api/v1/index_cn.rst | 2 +- doc/api/v2/config/layer.rst | 5 ++++ .../paddle/trainer_config_helpers/layers.py | 25 ++++++++++++++++- .../tests/configs/file_list.sh | 2 +- .../protostr/test_resize_layer.protostr | 27 +++++++++++++++++++ .../tests/configs/test_resize_layer.py | 6 +++++ 6 files changed, 64 insertions(+), 3 deletions(-) create mode 100644 python/paddle/trainer_config_helpers/tests/configs/protostr/test_resize_layer.protostr create mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_resize_layer.py diff --git a/doc/api/v1/index_cn.rst b/doc/api/v1/index_cn.rst index 3718cd73a2..cf146dc088 100644 --- a/doc/api/v1/index_cn.rst +++ b/doc/api/v1/index_cn.rst @@ -21,7 +21,7 @@ Model Config API trainer_config_helpers/optimizers.rst trainer_config_helpers/data_sources.rst trainer_config_helpers/layers.rst - trainer_config_helpers/activations.rst + trainer_config_helpers/activations.rst trainer_config_helpers/poolings.rst trainer_config_helpers/networks.rst trainer_config_helpers/evaluators.rst diff --git a/doc/api/v2/config/layer.rst b/doc/api/v2/config/layer.rst index c94627a728..d4e9d53e5c 100644 --- a/doc/api/v2/config/layer.rst +++ b/doc/api/v2/config/layer.rst @@ -345,6 +345,11 @@ clip .. autoclass:: paddle.v2.layer.clip :noindex: +resize +------ +.. autoclass:: paddle.v2.layer.resize + :noindex: + slope_intercept --------------- .. autoclass:: paddle.v2.layer.slope_intercept diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index 74025d2a7b..d37f29d2c4 100644 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -142,6 +142,7 @@ __all__ = [ 'img_pool3d_layer', 'scale_shift_layer', 'img_conv3d_layer', + 'resize_layer', ] @@ -250,6 +251,8 @@ class LayerType(object): KMAX_SEQ_SCORE = 'kmax_seq_score' SCALE_SHIFT_LAYER = 'scale_shift' + RESIZE = 'resize' + @staticmethod def is_layer_type(type_name): """ @@ -6473,7 +6476,7 @@ def switch_order_layer(input, act=None, layer_attr=None): """ - This layer switch dimension order of image input. + This layer switch dimension order of image input. From order "batchSize, channels, height, width" to order "batchSize, height, width, channels". @@ -6932,3 +6935,23 @@ def scale_shift_layer(input, name=None, param_attr=None, bias_attr=None): bias=ParamAttr.to_bias(bias_attr)) return LayerOutput( name, LayerType.SCALE_SHIFT_LAYER, parents=[input], size=input.size) + + +@wrap_name_default("resize") +def resize_layer(input, size, name=None): + """ + The resize layer resizes the input matrix with a shape of [Height, Width] + into the output matrix with a shape of [Height x Width / size, size], + where size is the parameter of this layer indicating the output dimension. + + :param input: The input to this layer. + :type input: LayerOutput. + :param name: The name of this layer. It is optional. + :type name: basestring + :param size: The resized output dimesion of this layer. + :type size: int + :return: A LayerOutput object. + :rtype: LayerOutput + """ + Layer(name=name, type=LayerType.RESIZE, inputs=Input(input.name), size=size) + return LayerOutput(name, LayerType.RESIZE, parents=[input], size=input.size) diff --git a/python/paddle/trainer_config_helpers/tests/configs/file_list.sh b/python/paddle/trainer_config_helpers/tests/configs/file_list.sh index 8a204a96f3..6a4550c209 100755 --- a/python/paddle/trainer_config_helpers/tests/configs/file_list.sh +++ b/python/paddle/trainer_config_helpers/tests/configs/file_list.sh @@ -10,6 +10,6 @@ test_prelu_layer test_row_conv test_detection_output_layer test_multibox_loss_la test_recursive_topology test_gated_unit_layer test_clip_layer test_row_l2_norm_layer test_kmax_seq_socre_layer test_sub_nested_seq_select_layer test_scale_shift_layer test_seq_slice_layer test_cross_entropy_over_beam test_pooling3D_layer -test_conv3d_layer test_deconv3d_layer test_BatchNorm3D) +test_conv3d_layer test_deconv3d_layer test_BatchNorm3D test_resize_layer) export whole_configs=(test_split_datasource) diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_resize_layer.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_resize_layer.protostr new file mode 100644 index 0000000000..9399252b23 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_resize_layer.protostr @@ -0,0 +1,27 @@ +type: "nn" +layers { + name: "input" + type: "data" + size: 300 + active_type: "" +} +layers { + name: "__resize_0__" + type: "resize" + size: 150 + active_type: "" + inputs { + input_layer_name: "input" + } +} +input_layer_names: "input" +output_layer_names: "__resize_0__" +sub_models { + name: "root" + layer_names: "input" + layer_names: "__resize_0__" + input_layer_names: "input" + output_layer_names: "__resize_0__" + is_recurrent_layer_group: false +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_resize_layer.py b/python/paddle/trainer_config_helpers/tests/configs/test_resize_layer.py new file mode 100644 index 0000000000..09a6f50733 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/test_resize_layer.py @@ -0,0 +1,6 @@ +from paddle.trainer_config_helpers import * + +data = data_layer(name='input', size=300) +resized = resize_layer(input=data, size=150) + +outputs(resized) From 04e604b7198179d2feedd76b2cf455656698b21f Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Sat, 30 Sep 2017 16:55:40 -0700 Subject: [PATCH 081/170] Unify Map in OpDescBind --- paddle/framework/op_desc.cc | 27 ++++++++++++++++++++++++++- paddle/framework/op_desc.h | 37 ++++++------------------------------- paddle/platform/enforce.h | 4 ++-- 3 files changed, 34 insertions(+), 34 deletions(-) diff --git a/paddle/framework/op_desc.cc b/paddle/framework/op_desc.cc index 0c12c55dc0..33a064890c 100644 --- a/paddle/framework/op_desc.cc +++ b/paddle/framework/op_desc.cc @@ -112,6 +112,30 @@ const std::unordered_map &OpDescBind::GetAttrMap() return attrs_; } +struct SetAttrDescVisitor : public boost::static_visitor { + explicit SetAttrDescVisitor(OpDesc::Attr *attr) : attr_(attr) {} + mutable OpDesc::Attr *attr_; + void operator()(int v) const { attr_->set_i(v); } + void operator()(float v) const { attr_->set_f(v); } + void operator()(const std::string &v) const { attr_->set_s(v); } + void operator()(bool b) const { attr_->set_b(b); } + + void operator()(const std::vector &v) const { + VectorToRepeated(v, attr_->mutable_ints()); + } + void operator()(const std::vector &v) const { + VectorToRepeated(v, attr_->mutable_floats()); + } + void operator()(const std::vector &v) const { + VectorToRepeated(v, attr_->mutable_strings()); + } + void operator()(const std::vector &v) const { + VectorToRepeated(v, attr_->mutable_bools()); + } + void operator()(BlockDesc *desc) const { attr_->set_block_idx(desc->idx()); } + void operator()(boost::blank) const { PADDLE_THROW("Unexpected branch"); } +}; + void OpDescBind::Sync() { if (need_update_) { this->op_desc_.mutable_inputs()->Clear(); @@ -134,7 +158,8 @@ void OpDescBind::Sync() { attr_desc->set_name(attr.first); attr_desc->set_type( static_cast(attr.second.which() - 1)); - boost::apply_visitor(SetAttrDescVisitor(attr_desc), attr.second); + SetAttrDescVisitor visitor(attr_desc); + boost::apply_visitor(visitor, attr.second); } need_update_ = false; diff --git a/paddle/framework/op_desc.h b/paddle/framework/op_desc.h index 0cf7d13971..e03b4d067f 100644 --- a/paddle/framework/op_desc.h +++ b/paddle/framework/op_desc.h @@ -17,6 +17,7 @@ limitations under the License. */ #include #include #include "paddle/framework/attribute.h" +#include "paddle/framework/op_info.h" #include "paddle/framework/var_desc.h" namespace paddle { @@ -61,48 +62,22 @@ class OpDescBind { void SetBlockAttr(const std::string &name, BlockDescBind &block); // Only be used in C++ - void SetAttrMap(const std::unordered_map &attr_map); + void SetAttrMap(const AttributeMap &attr_map); Attribute GetAttr(const std::string &name) const; int GetBlockAttr(const std::string &name) const; // Only be used in C++ - const std::unordered_map &GetAttrMap() const; + const AttributeMap &GetAttrMap() const; private: - struct SetAttrDescVisitor : public boost::static_visitor { - explicit SetAttrDescVisitor(OpDesc::Attr *attr) : attr_(attr) {} - mutable OpDesc::Attr *attr_; - void operator()(int v) const { attr_->set_i(v); } - void operator()(float v) const { attr_->set_f(v); } - void operator()(const std::string &v) const { attr_->set_s(v); } - void operator()(bool b) const { attr_->set_b(b); } - - void operator()(const std::vector &v) const { - VectorToRepeated(v, attr_->mutable_ints()); - } - void operator()(const std::vector &v) const { - VectorToRepeated(v, attr_->mutable_floats()); - } - void operator()(const std::vector &v) const { - VectorToRepeated(v, attr_->mutable_strings()); - } - void operator()(const std::vector &v) const { - VectorToRepeated(v, attr_->mutable_bools()); - } - void operator()(BlockDesc *desc) const { - attr_->set_block_idx(desc->idx()); - } - void operator()(boost::blank) const { PADDLE_THROW("Unexpected branch"); } - }; - void Sync(); OpDesc op_desc_; - std::unordered_map> inputs_; - std::unordered_map> outputs_; - std::unordered_map attrs_; + VariableNameMap inputs_; + VariableNameMap outputs_; + AttributeMap attrs_; // need_update_ indicate there some local changes not be synchronized. If // local changes should be synchronized, need_update_ should be set to true. diff --git a/paddle/platform/enforce.h b/paddle/platform/enforce.h index b523ef03c0..52bd23039b 100644 --- a/paddle/platform/enforce.h +++ b/paddle/platform/enforce.h @@ -185,7 +185,7 @@ inline void throw_on_error(T e) { std::make_exception_ptr( \ std::runtime_error(paddle::string::Sprintf(__VA_ARGS__))), \ __FILE__, __LINE__); \ - } while (0) + } while (false) #define PADDLE_ENFORCE(...) \ do { \ @@ -195,7 +195,7 @@ inline void throw_on_error(T e) { throw ::paddle::platform::EnforceNotMet(std::current_exception(), \ __FILE__, __LINE__); \ } \ - } while (0) + } while (false) /* * Some enforce helpers here, usage: From c7ae0aac6660cc6cad2e7977ae573359433d484c Mon Sep 17 00:00:00 2001 From: kexinzhao <19hskevin87@gmail.com> Date: Sun, 1 Oct 2017 09:36:02 -0700 Subject: [PATCH 082/170] Fix typo in new_op_cn.md --- doc/howto/dev/new_op_cn.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/howto/dev/new_op_cn.md b/doc/howto/dev/new_op_cn.md index 9d3d02ffc3..c823d7e9fc 100644 --- a/doc/howto/dev/new_op_cn.md +++ b/doc/howto/dev/new_op_cn.md @@ -206,7 +206,7 @@ MulOp(const std::string &type, const framework::VariableNameMap &inputs, - `REGISTER_OP` ： 注册`ops::MulOp`类，类型名为`mul`，该类的`ProtoMaker`为`ops::MulOpMaker`，注册`ops::MulOpGrad`，类型名为`mul_grad`。 - `REGISTER_OP_WITHOUT_GRADIENT` ： 用于注册没有反向的Op。 - - `REGISTER_OP_CPU_KERNEL` ：注册`ops::MulKernel`类，并特化模板参数为`paddle::platform::CPUPlace`和`float`类型，同理，注册`ops::MulKernel`类。 + - `REGISTER_OP_CPU_KERNEL` ：注册`ops::MulKernel`类，并特化模板参数为`paddle::platform::CPUPlace`和`float`类型，同理，注册`ops::MulGradKernel`类。 - 在 `.cu`文件中注册GPU Kernel。 From 83764d491cc8a835a315f29a04fe1addfd1a05ae Mon Sep 17 00:00:00 2001 From: kexinzhao <19hskevin87@gmail.com> Date: Sun, 1 Oct 2017 09:40:49 -0700 Subject: [PATCH 083/170] Update new_op_en.md --- doc/howto/dev/new_op_en.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/howto/dev/new_op_en.md b/doc/howto/dev/new_op_en.md index 57ff7caad1..1e88e1f5b4 100644 --- a/doc/howto/dev/new_op_en.md +++ b/doc/howto/dev/new_op_en.md @@ -205,7 +205,7 @@ The definition of its corresponding backward operator, if applicable, is similar - `REGISTER_OP` registers the `ops::MulOp` class, type named `mul`, its type `ProtoMaker` is `ops::MulOpMaker`, registering `ops::MulOpGrad` as `mul_grad`. - `REGISTER_OP_WITHOUT_GRADIENT` registers an operator without gradient. - - `REGISTER_OP_CPU_KERNEL` registers `ops::MulKernel` class and specialized template types `paddle::platform::CPUPlace` and `float`, which also registers `ops::MulKernel`. + - `REGISTER_OP_CPU_KERNEL` registers `ops::MulKernel` class and specialized template types `paddle::platform::CPUPlace` and `float`, which also registers `ops::MulGradKernel`. - Registering GPU Kernel in `.cu` files From 18799476c20743313eb8361efc6cd21886d95862 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Mon, 2 Oct 2017 11:39:10 -0700 Subject: [PATCH 084/170] Use `type_defs.h` to resolve cyclic dependencies --- paddle/framework/attribute.h | 10 +--------- paddle/framework/op_desc.h | 2 +- paddle/framework/op_info.h | 7 +------ paddle/framework/type_defs.h | 38 ++++++++++++++++++++++++++++++++++++ 4 files changed, 41 insertions(+), 16 deletions(-) create mode 100644 paddle/framework/type_defs.h diff --git a/paddle/framework/attribute.h b/paddle/framework/attribute.h index c7559cefb6..d13530e340 100644 --- a/paddle/framework/attribute.h +++ b/paddle/framework/attribute.h @@ -21,20 +21,12 @@ limitations under the License. */ #include #include "paddle/framework/framework.pb.h" +#include "paddle/framework/type_defs.h" #include "paddle/platform/enforce.h" -#include "paddle/platform/variant.h" namespace paddle { namespace framework { -// The order should be as same as framework.proto -typedef boost::variant, - std::vector, std::vector, bool, - std::vector, BlockDesc*> - Attribute; - -typedef std::unordered_map AttributeMap; - ProgramDesc& GetProgramDesc(); template diff --git a/paddle/framework/op_desc.h b/paddle/framework/op_desc.h index e03b4d067f..0af4169715 100644 --- a/paddle/framework/op_desc.h +++ b/paddle/framework/op_desc.h @@ -17,7 +17,7 @@ limitations under the License. */ #include #include #include "paddle/framework/attribute.h" -#include "paddle/framework/op_info.h" +#include "paddle/framework/type_defs.h" #include "paddle/framework/var_desc.h" namespace paddle { diff --git a/paddle/framework/op_info.h b/paddle/framework/op_info.h index 6d1ee4dece..470336d367 100644 --- a/paddle/framework/op_info.h +++ b/paddle/framework/op_info.h @@ -19,15 +19,10 @@ #include #include "paddle/framework/attribute.h" #include "paddle/framework/op_desc.h" +#include "paddle/framework/type_defs.h" namespace paddle { namespace framework { -class OperatorBase; -using VariableNameMap = std::map>; - -using OpCreator = std::function; class GradOpDescMakerBase { public: diff --git a/paddle/framework/type_defs.h b/paddle/framework/type_defs.h new file mode 100644 index 0000000000..dec5066f1e --- /dev/null +++ b/paddle/framework/type_defs.h @@ -0,0 +1,38 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#pragma once +#include +#include +#include "paddle/platform/variant.h" + +namespace paddle { +namespace framework { +class OperatorBase; +using VariableNameMap = std::map>; + +// The order should be as same as framework.proto +using Attribute = + boost::variant, + std::vector, std::vector, bool, + std::vector, BlockDesc*>; + +using AttributeMap = std::unordered_map; + +using OpCreator = std::function; + +} // namespace framework +} // namespace paddle From 31bdb3f3cc98a38a3ec4951f26dd06260cdc4695 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Mon, 2 Oct 2017 14:03:38 -0700 Subject: [PATCH 085/170] tmp --- paddle/framework/operator.h | 98 +++++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/paddle/framework/operator.h b/paddle/framework/operator.h index 73e53a4176..f807909650 100644 --- a/paddle/framework/operator.h +++ b/paddle/framework/operator.h @@ -317,6 +317,104 @@ class ExecutionContext : public InferShapeContext { const platform::DeviceContext& device_context_; }; +class CompileTimeInferShapeContext : public InferShapeContextBase { + public: + CompileTimeInferShapeContext(const OperatorBase& op, const Scope& scope) + : op_(op), scope_(scope) {} + + bool HasInput(const std::string& name) const { + auto ipt = op_.Input(name); + auto* var = ipt == kEmptyVarName ? nullptr : scope_.FindVar(ipt); + return var != nullptr; + } + + bool HasOutput(const std::string& name) const { + auto ipt = op_.Output(name); + auto* var = ipt == kEmptyVarName ? nullptr : scope_.FindVar(ipt); + return var != nullptr; + } + + bool HasInputs(const std::string& name) const { + auto inputs = op_.Inputs(name); + if (inputs.size() == 0UL) { + return false; + } + for (auto& input : inputs) { + if (scope_.FindVar(input) == nullptr) { + return false; + } + } + return true; + } + + bool HasOutputs(const std::string& name) const { + auto outputs = op_.Outputs(name); + if (outputs.size() == 0UL) { + return false; + } + for (auto& output : outputs) { + if (scope_.FindVar(output) == nullptr) { + return false; + } + } + return true; + } + + DDim GetInputDim(const std::string& name) const { + return GetDim(op_.Input(name)); + } + + void SetInputDim(const std::string& name, const DDim& dim) { + SetDim(op_.Input(name), dim); + } + + DDim GetOutputDim(const std::string& name) const { + return GetDim(op_.Output(name)); + } + + void SetOutputDim(const std::string& name, const DDim& dim) { + SetDim(op_.Output(name), dim); + } + + AttrReader Attrs() const { return AttrReader(op_.Attrs()); } + + const std::vector& Inputs(const std::string& name) const { + return op_.Inputs(name); + } + + const std::vector& Outputs(const std::string& name) const { + return op_.Outputs(name); + } + + private: + template + Tensor* GetTensor(const std::string& name) const { + Tensor* t = nullptr; + auto* var = scope_.FindVar(name); + if (!var->IsType() && !var->IsType()) { + if (Allocate) { + t = var->GetMutable(); + } else { + PADDLE_THROW("Variable(%s) should be tensor", name); + } + } else { + t = GetTensorFromVar(scope_.FindVar(name)); + } + return t; + } + + DDim GetDim(const std::string& name) const { + return GetTensor(name)->dims(); + } + + void SetDim(const std::string& name, const DDim& dim) { + GetTensor(name)->Resize(dim); + } + + const OperatorBase& op_; + const Scope& scope_; +}; + class RuntimeInferShapeContext : public InferShapeContextBase { public: RuntimeInferShapeContext(const OperatorBase& op, const Scope& scope) From 1ac654a69f3713cf12d47aff0b853b63de354803 Mon Sep 17 00:00:00 2001 From: Kexin Zhao Date: Mon, 2 Oct 2017 13:48:35 -0700 Subject: [PATCH 086/170] Implementing the Adagrad optimizer step operator --- paddle/operators/adagrad_op.cc | 85 +++++++++++++++++++ paddle/operators/adagrad_op.cu | 20 +++++ paddle/operators/adagrad_op.h | 53 ++++++++++++ .../v2/framework/tests/test_adagrad_op.py | 32 +++++++ 4 files changed, 190 insertions(+) create mode 100644 paddle/operators/adagrad_op.cc create mode 100644 paddle/operators/adagrad_op.cu create mode 100644 paddle/operators/adagrad_op.h create mode 100644 python/paddle/v2/framework/tests/test_adagrad_op.py diff --git a/paddle/operators/adagrad_op.cc b/paddle/operators/adagrad_op.cc new file mode 100644 index 0000000000..03e22cc600 --- /dev/null +++ b/paddle/operators/adagrad_op.cc @@ -0,0 +1,85 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/operators/adagrad_op.h" + +namespace paddle { +namespace operators { + +class AdagradOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(framework::InferShapeContextBase *ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("param"), + "Input(param) of AdagradOp should not be null."); + PADDLE_ENFORCE(ctx->HasInput("grad"), + "Input(grad) of AdagradOp should not be null."); + PADDLE_ENFORCE(ctx->HasInput("moment"), + "Input(moment) of AdagradOp should not be null."); + + PADDLE_ENFORCE(ctx->HasOutput("param_out"), + "Output(param_out) of AdagradOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("moment_out"), + "Output(moment_out) of AdagradOp should not be null."); + + auto param_dim = ctx->GetInputDim("param"); + PADDLE_ENFORCE_EQ( + param_dim, ctx->GetInputDim("grad"), + "Param and grad input of AdagradOp should have the same dimension."); + PADDLE_ENFORCE_EQ( + param_dim, ctx->GetInputDim("moment"), + "Param and moment input of AdagradOp should have the same dimension."); + + ctx->SetOutputDim("param_out", param_dim); + ctx->SetOutputDim("moment_out", param_dim); + } +}; + +class AdagradOpMaker : public framework::OpProtoAndCheckerMaker { + public: + AdagradOpMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("param", "Input parameter"); + AddInput("grad", "Input gradient"); + AddInput("moment", "Second moment"); + + AddOutput("param_out", "Output parameter"); + AddOutput("moment_out", "Output second moment"); + + AddAttr("learning_rate", "Learning rate"); + AddAttr("epsilon", "Constant for numerical stability"); + AddComment(R"DOC( + +Adaptive Gradient Algorithm (Adagrad). + +moment_out = moment + grad * grad +param_out = param - learning_rate * grad / (sqrt(moment_out) + epsilon) + +The original paper(http://www.jmlr.org/papers/volume12/duchi11a/duchi11a.pdf) +does not have the epsilon attribute. It is added here for numerical stability +by avoiding division by zero. + +)DOC"); + } +}; +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP_WITHOUT_GRADIENT(adagrad, ops::AdagradOp, ops::AdagradOpMaker); +REGISTER_OP_CPU_KERNEL(adagrad, + ops::AdagradOpKernel); diff --git a/paddle/operators/adagrad_op.cu b/paddle/operators/adagrad_op.cu new file mode 100644 index 0000000000..be16973c54 --- /dev/null +++ b/paddle/operators/adagrad_op.cu @@ -0,0 +1,20 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#define EIGEN_USE_GPU +#include "paddle/operators/adagrad_op.h" + +namespace ops = paddle::operators; +REGISTER_OP_GPU_KERNEL(adagrad, + ops::AdagradOpKernel); \ No newline at end of file diff --git a/paddle/operators/adagrad_op.h b/paddle/operators/adagrad_op.h new file mode 100644 index 0000000000..ca1836c3fa --- /dev/null +++ b/paddle/operators/adagrad_op.h @@ -0,0 +1,53 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once +#include "paddle/framework/eigen.h" +#include "paddle/framework/op_registry.h" + +namespace paddle { +namespace operators { + +using Tensor = framework::Tensor; +template +using EigenVector = framework::EigenVector; + +template +class AdagradOpKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const override { + auto param_out = ctx.Output("param_out"); + auto moment_out = ctx.Output("moment_out"); + + param_out->mutable_data(ctx.GetPlace()); + moment_out->mutable_data(ctx.GetPlace()); + + float lr = ctx.Attr("learning_rate"); + float epsilon = ctx.Attr("epsilon"); + + auto p = EigenVector::Flatten(*ctx.Input("param")); + auto g = EigenVector::Flatten(*ctx.Input("grad")); + auto m = EigenVector::Flatten(*ctx.Input("moment")); + auto p_out = EigenVector::Flatten(*param_out); + auto m_out = EigenVector::Flatten(*moment_out); + auto place = ctx.GetEigenDevice(); + + m_out.device(place) = m + g * g; + p_out.device(place) = p - lr * g / (m_out.sqrt() + epsilon); + } +}; + +} // namespace operators +} // namespace paddle diff --git a/python/paddle/v2/framework/tests/test_adagrad_op.py b/python/paddle/v2/framework/tests/test_adagrad_op.py new file mode 100644 index 0000000000..b3f8b812e1 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_adagrad_op.py @@ -0,0 +1,32 @@ +import unittest +import numpy as np +from op_test import OpTest + + +class TestAdagradOp(OpTest): + def setUp(self): + self.op_type = "adagrad" + + param = np.random.random((123, 321)).astype("float32") + grad = np.random.random((123, 321)).astype("float32") + moment = np.zeros((123, 321)).astype("float32") + + learning_rate = 0.01 + epsilon = 1e-6 + + self.inputs = {'param': param, 'grad': grad, 'moment': moment} + + self.attrs = {'learning_rate': learning_rate, 'epsilon': epsilon} + + moment_out = moment + grad * grad + param_out = param - learning_rate * grad / (np.sqrt(moment_out) + + epsilon) + + self.outputs = {'param_out': param_out, 'moment_out': moment_out} + + def test_check_output(self): + self.check_output() + + +if __name__ == "__main__": + unittest.main() From 37bbaabdf1abfe14e19cf7dcb7a842a10b36d1c8 Mon Sep 17 00:00:00 2001 From: dongzhihong Date: Mon, 2 Oct 2017 14:17:17 -0700 Subject: [PATCH 087/170] "fix conflict" --- paddle/framework/op_info.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/framework/op_info.h b/paddle/framework/op_info.h index 7940922b09..9672e540c8 100644 --- a/paddle/framework/op_info.h +++ b/paddle/framework/op_info.h @@ -20,7 +20,7 @@ #include "paddle/framework/attribute.h" #include "paddle/framework/op_desc.h" #include "paddle/framework/type_defs.h" - +#include "paddle/platform/macros.h" namespace paddle { namespace framework { From 6b051b651ae72305d9877fd3cd094028c21bdddb Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Mon, 2 Oct 2017 14:24:03 -0700 Subject: [PATCH 088/170] optimize code --- paddle/operators/recurrent_op.cc | 38 ++++++++++++---------- paddle/operators/recurrent_op.h | 4 +-- paddle/operators/rnn/recurrent_op_utils.cc | 8 ++--- 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/paddle/operators/recurrent_op.cc b/paddle/operators/recurrent_op.cc index 016e2043fd..bcd6a3410a 100644 --- a/paddle/operators/recurrent_op.cc +++ b/paddle/operators/recurrent_op.cc @@ -32,24 +32,25 @@ void RecurrentAlgorithm::Run(const Scope& scope, const platform::DeviceContext& dev_ctx) const { auto* input0 = scope.FindVar(arg_->inlinks[0]); PADDLE_ENFORCE_NOT_NULL(input0); - seq_len_ = input0->GetMutable()->dims()[0]; - PADDLE_ENFORCE_GT(seq_len_, 0); + size_t seq_len = input0->GetMutable()->dims()[0]; + PADDLE_ENFORCE_GT(seq_len, 0); - CreateScopes(scope); + CreateScopes(scope, seq_len); auto& step_scopes = GetStepScopes(scope); - rnn::SegmentInputs(step_scopes, arg_->inlinks, seq_len_); + rnn::SegmentInputs(step_scopes, arg_->inlinks, seq_len); InitMemories(step_scopes[0]); - for (size_t i = 0; i < seq_len_; i++) { - if (i > 0) { - rnn::LinkMemories(step_scopes, arg_->memories, i, -1); + for (size_t step_id = 0; step_id < seq_len; step_id++) { + if (step_id > 0) { + rnn::LinkMemories(step_scopes, arg_->memories, step_id, -1); } - (*stepnet_)->Run(*step_scopes[i], dev_ctx); + (*stepnet_)->Run(*step_scopes[step_id], dev_ctx); } - rnn::ConcatOutputs(step_scopes, arg_->outlinks, seq_len_); + rnn::ConcatOutputs(step_scopes, arg_->outlinks, seq_len); } -void RecurrentAlgorithm::CreateScopes(const Scope& scope) const { +void RecurrentAlgorithm::CreateScopes(const Scope& scope, + size_t seq_len) const { // TODO(superjom) Only two scopes are needed for inference, this case will be // supported later. auto step_scopes_var = scope.FindVar(arg_->step_scopes); @@ -60,8 +61,8 @@ void RecurrentAlgorithm::CreateScopes(const Scope& scope) const { PADDLE_ENFORCE_NOT_NULL(stepnet_); PADDLE_ENFORCE(!(*stepnet_)->Outputs().empty(), "stepnet_ op has no outputs"); - if (seq_len_ > step_scopes->size()) { - for (size_t i = step_scopes->size(); i < seq_len_; ++i) { + if (seq_len > step_scopes->size()) { + for (size_t i = step_scopes->size(); i < seq_len; ++i) { auto& step_scope = scope.NewScope(); // create step net's temp inputs @@ -144,17 +145,18 @@ class RecurrentAlgorithmProtoAndCheckerMaker void RecurrentGradientAlgorithm::Run( const Scope& scope, const platform::DeviceContext& dev_ctx) const { - seq_len_ = - scope.FindVar(arg_->inlinks[0])->GetMutable()->dims()[0]; + auto* input0 = scope.FindVar(arg_->inlinks[0]); + PADDLE_ENFORCE_NOT_NULL(input0); + size_t seq_len = input0->GetMutable()->dims()[0]; auto step_scopes = GetStepScopes(scope); - rnn::SegmentInputs(step_scopes, arg_->inlinks, seq_len_); - for (int step_id = seq_len_ - 1; step_id >= 0; --step_id) { - if (static_cast(step_id) != seq_len_ - 1) { + rnn::SegmentInputs(step_scopes, arg_->inlinks, seq_len); + for (int step_id = seq_len - 1; step_id >= 0; --step_id) { + if (step_id != seq_len - 1) { rnn::LinkMemories(step_scopes, arg_->memories, step_id, 1); } (*stepnet_)->Run(*step_scopes[step_id], dev_ctx); } - rnn::ConcatOutputs(step_scopes, arg_->outlinks, seq_len_); + rnn::ConcatOutputs(step_scopes, arg_->outlinks, seq_len); LinkBootMemoryGradients(step_scopes[0]); } diff --git a/paddle/operators/recurrent_op.h b/paddle/operators/recurrent_op.h index 752025e42c..253d7e3284 100644 --- a/paddle/operators/recurrent_op.h +++ b/paddle/operators/recurrent_op.h @@ -48,7 +48,7 @@ class RecurrentAlgorithm { * NOTE the scopes are reused in both the forward and backward, so just * create once and expand its size if more steps need. */ - void CreateScopes(const framework::Scope& scope) const; + void CreateScopes(const framework::Scope& scope, size_t seq_len) const; const std::vector& GetStepScopes( const framework::Scope& scope) const { @@ -61,7 +61,6 @@ class RecurrentAlgorithm { private: std::unique_ptr* stepnet_; rnn::Argument* arg_; - mutable size_t seq_len_; }; class RecurrentGradientAlgorithm { @@ -97,7 +96,6 @@ class RecurrentGradientAlgorithm { private: rnn::Argument* arg_; - mutable size_t seq_len_; std::unique_ptr* stepnet_; }; diff --git a/paddle/operators/rnn/recurrent_op_utils.cc b/paddle/operators/rnn/recurrent_op_utils.cc index a02994f99d..a37d21d480 100644 --- a/paddle/operators/rnn/recurrent_op_utils.cc +++ b/paddle/operators/rnn/recurrent_op_utils.cc @@ -53,12 +53,12 @@ void ConcatOutputs(const std::vector& step_scopes, const std::vector& outlinks, const size_t seq_len) { for (size_t i = 0; i < outlinks.size(); i++) { - auto output_var = step_scopes[0]->parent().FindVar(outlinks[i]); + auto* output_var = step_scopes[0]->parent().FindVar(outlinks[i]); PADDLE_ENFORCE_NOT_NULL(output_var, "output link [%s] is not in scope.", outlinks[i]); LoDTensor* output = output_var->GetMutable(); - auto step_scope_var = step_scopes[0]->FindVar(outlinks[i]); + auto* step_scope_var = step_scopes[0]->FindVar(outlinks[i]); PADDLE_ENFORCE_NOT_NULL(step_scope_var, "%s not in scope", outlinks[i]); f::DDim step_dims = step_scope_var->template GetMutable()->dims(); @@ -89,8 +89,8 @@ void LinkMemories(const std::vector& scopes, step_id + offset, scopes.size(), "offset [%d] is out of range, it must be less than (%d - %d)", offset, scopes.size(), step_id); - auto scope = scopes[step_id]; - auto linked_scope = scopes[step_id + offset]; + auto* scope = scopes[step_id]; + auto* linked_scope = scopes[step_id + offset]; for (auto& attr : memories) { auto mem = scope->FindVar(attr.pre_var)->GetMutable(); auto linked_mem = linked_scope->FindVar(attr.var)->GetMutable(); From cde542e6524d8fd084983e20e0051a3caf22f6b1 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Mon, 2 Oct 2017 14:51:15 -0700 Subject: [PATCH 089/170] optimize auto --- paddle/operators/recurrent_op.cc | 6 +++--- paddle/operators/rnn/recurrent_op_utils.cc | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/paddle/operators/recurrent_op.cc b/paddle/operators/recurrent_op.cc index bcd6a3410a..04c4c24951 100644 --- a/paddle/operators/recurrent_op.cc +++ b/paddle/operators/recurrent_op.cc @@ -53,9 +53,9 @@ void RecurrentAlgorithm::CreateScopes(const Scope& scope, size_t seq_len) const { // TODO(superjom) Only two scopes are needed for inference, this case will be // supported later. - auto step_scopes_var = scope.FindVar(arg_->step_scopes); + auto* step_scopes_var = scope.FindVar(arg_->step_scopes); PADDLE_ENFORCE(step_scopes_var != nullptr, ""); - auto step_scopes = step_scopes_var->GetMutable>(); + auto* step_scopes = step_scopes_var->GetMutable>(); // Now all variables in scope must be created outside of op. PADDLE_ENFORCE_NOT_NULL(stepnet_); @@ -148,7 +148,7 @@ void RecurrentGradientAlgorithm::Run( auto* input0 = scope.FindVar(arg_->inlinks[0]); PADDLE_ENFORCE_NOT_NULL(input0); size_t seq_len = input0->GetMutable()->dims()[0]; - auto step_scopes = GetStepScopes(scope); + auto& step_scopes = GetStepScopes(scope); rnn::SegmentInputs(step_scopes, arg_->inlinks, seq_len); for (int step_id = seq_len - 1; step_id >= 0; --step_id) { if (step_id != seq_len - 1) { diff --git a/paddle/operators/rnn/recurrent_op_utils.cc b/paddle/operators/rnn/recurrent_op_utils.cc index a37d21d480..ef317a71f1 100644 --- a/paddle/operators/rnn/recurrent_op_utils.cc +++ b/paddle/operators/rnn/recurrent_op_utils.cc @@ -92,8 +92,8 @@ void LinkMemories(const std::vector& scopes, auto* scope = scopes[step_id]; auto* linked_scope = scopes[step_id + offset]; for (auto& attr : memories) { - auto mem = scope->FindVar(attr.pre_var)->GetMutable(); - auto linked_mem = linked_scope->FindVar(attr.var)->GetMutable(); + auto* mem = scope->FindVar(attr.pre_var)->GetMutable(); + auto* linked_mem = linked_scope->FindVar(attr.var)->GetMutable(); mem->Resize(linked_mem->dims()); mem->ShareDataWith(*linked_mem); } @@ -106,11 +106,11 @@ void InitArgument(const ArgumentName& name, Argument* arg, arg->inlinks = op.Inputs(name.inlinks); arg->outlinks = op.Outputs(name.outlinks); - auto boot_memories = + auto& boot_memories = is_grad ? op.Outputs(name.boot_memories) : op.Inputs(name.boot_memories); // attributes - auto memories = op.Attr>(name.memories); - auto pre_memories = op.Attr>(name.pre_memories); + auto& memories = op.Attr>(name.memories); + auto& pre_memories = op.Attr>(name.pre_memories); PADDLE_ENFORCE(memories.size() == boot_memories.size(), "the size of memories, boot_memories don't match:%d,%d", From 05cbd4daacb699ec94d77f5b0229aaaa16c35af1 Mon Sep 17 00:00:00 2001 From: Kexin Zhao Date: Mon, 2 Oct 2017 15:00:41 -0700 Subject: [PATCH 090/170] fix format --- paddle/operators/adagrad_op.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/operators/adagrad_op.cu b/paddle/operators/adagrad_op.cu index be16973c54..a5b7951121 100644 --- a/paddle/operators/adagrad_op.cu +++ b/paddle/operators/adagrad_op.cu @@ -17,4 +17,4 @@ namespace ops = paddle::operators; REGISTER_OP_GPU_KERNEL(adagrad, - ops::AdagradOpKernel); \ No newline at end of file + ops::AdagradOpKernel); From d2bd6f45cb82531eff7ce7e64360d75e351c643d Mon Sep 17 00:00:00 2001 From: dongzhihong Date: Mon, 2 Oct 2017 15:07:02 -0700 Subject: [PATCH 091/170] "replace add with sum" --- paddle/framework/backward.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/framework/backward.cc b/paddle/framework/backward.cc index fbacfeed94..35759f8e78 100644 --- a/paddle/framework/backward.cc +++ b/paddle/framework/backward.cc @@ -159,7 +159,7 @@ static std::unique_ptr BackwardRecursive( insert_position.push_back( {dup_op.back(), OpRegistry::CreateOp( - "add", {{"X", {insert_add_x}}, {"Y", {insert_add_y}}}, + "add", {{"X", {insert_add_x}}, {"X", {insert_add_y}}}, {{"Out", {insert_add_out}}}, {})}); } } From 2bceab0fb446aa1c5370a613bf67041d4534f187 Mon Sep 17 00:00:00 2001 From: dzhwinter Date: Sun, 1 Oct 2017 08:29:09 +0000 Subject: [PATCH 092/170] add generic add operator --- paddle/framework/backward.cc | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/paddle/framework/backward.cc b/paddle/framework/backward.cc index 0ec18de5b8..c625c0caf7 100644 --- a/paddle/framework/backward.cc +++ b/paddle/framework/backward.cc @@ -141,9 +141,35 @@ static std::unique_ptr BackwardRecursive( net->ops_[op_offset]->Rename(name, dup_outputs.back()); } // collect all the offset to append `add` op for each alias - insert_position.push_back( - {dup_op.back(), OpRegistry::CreateOp("add", {{"X", {dup_outputs}}}, - {{"Out", {name}}}, {})}); + // + // one variable is shared between multiple operators. + // insert add operator one by one, then add it to output + if (dup_outputs.size() == 2) { + insert_position.push_back( + {dup_op.back(), + OpRegistry::CreateOp( + "add", {{"X", {dup_outputs[0]}}, {"Y", {dup_outputs[1]}}}, + {{"Out", {name}}}, {})}); + } else { + for (size_t output_idx = 0; output_idx < dup_outputs.size() - 1; + ++output_idx) { + auto insert_add_x = dup_outputs[output_idx]; + auto insert_add_y = dup_outputs[output_idx]; + auto insert_add_out = name + "@SHARED@" + std::to_string(output_idx); + // first add op inserted + if (output_idx == dup_outputs.size() - 1) { + insert_add_out = name; + } + if (output_idx != 0) { + insert_add_y = name + "@SHARED@" + std::to_string(output_idx); + } + insert_position.push_back( + {dup_op.back(), + OpRegistry::CreateOp( + "add", {{"X", {insert_add_x}}, {"Y", {insert_add_y}}}, + {{"Out", {insert_add_out}}}, {})}); + } + } } // make sure the inserted `add` ops follow the BFS order. From 800085fe2d37e5ad3ef706701ffb008a2c668ee1 Mon Sep 17 00:00:00 2001 From: dzhwinter Date: Sun, 1 Oct 2017 08:36:04 +0000 Subject: [PATCH 093/170] fix typo --- paddle/framework/backward.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/framework/backward.cc b/paddle/framework/backward.cc index c625c0caf7..b850939040 100644 --- a/paddle/framework/backward.cc +++ b/paddle/framework/backward.cc @@ -161,7 +161,7 @@ static std::unique_ptr BackwardRecursive( insert_add_out = name; } if (output_idx != 0) { - insert_add_y = name + "@SHARED@" + std::to_string(output_idx); + insert_add_y = name + "@SHARED@" + std::to_string(output_idx-1); } insert_position.push_back( {dup_op.back(), From f6496272cfcbb4d2ad9eb8a272a065007059a004 Mon Sep 17 00:00:00 2001 From: dongzhihong Date: Sun, 1 Oct 2017 09:18:21 -0700 Subject: [PATCH 094/170] relauch ci --- paddle/framework/backward.cc | 36 ++++++++++++++---------------------- 1 file changed, 14 insertions(+), 22 deletions(-) diff --git a/paddle/framework/backward.cc b/paddle/framework/backward.cc index b850939040..fbacfeed94 100644 --- a/paddle/framework/backward.cc +++ b/paddle/framework/backward.cc @@ -144,31 +144,23 @@ static std::unique_ptr BackwardRecursive( // // one variable is shared between multiple operators. // insert add operator one by one, then add it to output - if (dup_outputs.size() == 2) { + for (size_t output_idx = 0; output_idx < dup_outputs.size() - 1; + ++output_idx) { + auto insert_add_x = dup_outputs[output_idx]; + auto insert_add_y = dup_outputs[output_idx]; + auto insert_add_out = name + "@SHARED@" + std::to_string(output_idx); + // first add op inserted + if (output_idx == dup_outputs.size() - 2) { + insert_add_out = name; + } + if (output_idx != 0) { + insert_add_y = name + "@SHARED@" + std::to_string(output_idx - 1); + } insert_position.push_back( {dup_op.back(), OpRegistry::CreateOp( - "add", {{"X", {dup_outputs[0]}}, {"Y", {dup_outputs[1]}}}, - {{"Out", {name}}}, {})}); - } else { - for (size_t output_idx = 0; output_idx < dup_outputs.size() - 1; - ++output_idx) { - auto insert_add_x = dup_outputs[output_idx]; - auto insert_add_y = dup_outputs[output_idx]; - auto insert_add_out = name + "@SHARED@" + std::to_string(output_idx); - // first add op inserted - if (output_idx == dup_outputs.size() - 1) { - insert_add_out = name; - } - if (output_idx != 0) { - insert_add_y = name + "@SHARED@" + std::to_string(output_idx-1); - } - insert_position.push_back( - {dup_op.back(), - OpRegistry::CreateOp( - "add", {{"X", {insert_add_x}}, {"Y", {insert_add_y}}}, - {{"Out", {insert_add_out}}}, {})}); - } + "add", {{"X", {insert_add_x}}, {"Y", {insert_add_y}}}, + {{"Out", {insert_add_out}}}, {})}); } } From a1b935e356573266dfff08d5fd279815492c8843 Mon Sep 17 00:00:00 2001 From: dongzhihong Date: Mon, 2 Oct 2017 15:07:02 -0700 Subject: [PATCH 095/170] "replace add with sum" --- paddle/framework/backward.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/framework/backward.cc b/paddle/framework/backward.cc index fbacfeed94..35759f8e78 100644 --- a/paddle/framework/backward.cc +++ b/paddle/framework/backward.cc @@ -159,7 +159,7 @@ static std::unique_ptr BackwardRecursive( insert_position.push_back( {dup_op.back(), OpRegistry::CreateOp( - "add", {{"X", {insert_add_x}}, {"Y", {insert_add_y}}}, + "add", {{"X", {insert_add_x}}, {"X", {insert_add_y}}}, {{"Out", {insert_add_out}}}, {})}); } } From 578a357b616ee188d692764843ae834a449e81c2 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Mon, 2 Oct 2017 15:12:20 -0700 Subject: [PATCH 096/170] Make compile pass --- paddle/framework/CMakeLists.txt | 4 +--- paddle/framework/backward.cc | 33 +++++++++++++++++++++++++++++---- paddle/framework/op_desc.h | 14 ++++++++------ paddle/framework/op_registry.cc | 6 ++++++ paddle/framework/op_registry.h | 8 +++++--- 5 files changed, 49 insertions(+), 16 deletions(-) diff --git a/paddle/framework/CMakeLists.txt b/paddle/framework/CMakeLists.txt index 9140854a96..eb316b4c8c 100644 --- a/paddle/framework/CMakeLists.txt +++ b/paddle/framework/CMakeLists.txt @@ -26,10 +26,8 @@ cc_library(op_info SRCS op_info.cc DEPS attribute framework_proto proto_desc) cc_library(operator SRCS operator.cc DEPS op_info device_context tensor scope) cc_test(operator_test SRCS operator_test.cc DEPS operator op_registry) -cc_library(grad_op_builder SRCS grad_op_builder.cc DEPS operator proto_desc) -cc_library(op_registry SRCS op_registry.cc DEPS grad_op_builder op_proto_maker op_info) +cc_library(op_registry SRCS op_registry.cc DEPS op_proto_maker op_info operator) cc_test(op_registry_test SRCS op_registry_test.cc DEPS op_registry) -cc_test(grad_op_builder_test SRCS grad_op_builder_test.cc DEPS grad_op_builder op_registry sum_op) py_proto_compile(framework_py_proto SRCS framework.proto) # Generate an empty __init__.py to make framework_py_proto as a valid python module. diff --git a/paddle/framework/backward.cc b/paddle/framework/backward.cc index ab2567a25c..eb34bc3693 100644 --- a/paddle/framework/backward.cc +++ b/paddle/framework/backward.cc @@ -13,6 +13,7 @@ limitations under the License. */ #include "paddle/framework/backward.h" +#include "paddle/operators/net_op.h" #include #include @@ -24,6 +25,32 @@ namespace paddle { namespace framework { +static inline std::unique_ptr CreateGradOp( + const OperatorBase& op) { + OpDescBind op_desc; + op_desc.SetInputMap(op.Inputs()); + op_desc.SetOutputMap(op.Outputs()); + op_desc.SetType(op.Type()); + op_desc.SetAttrMap(op.Attrs()); + auto& info = OpInfoMap::Instance().Get(op.Type()); + auto grad_descs = info.grad_op_maker_(op_desc); + std::vector> grad_ops; + grad_ops.reserve(grad_descs.size()); + std::transform( + grad_descs.begin(), grad_descs.end(), std::back_inserter(grad_ops), + [](OpDescBind& grad_desc) { return OpRegistry::CreateOp(&grad_desc); }); + PADDLE_ENFORCE_GT(grad_ops.size(), 0); + if (grad_ops.size() == 1) { + return std::move(grad_ops[0]); + } else { + auto net_op = new operators::NetOp(); + for (auto& grad_op : grad_ops) { + net_op->AppendOp(std::move(grad_op)); + } + return std::unique_ptr(net_op); + } +} + template static void ForEachVarName(const Map& names, T callback) { for (auto& name : names) { @@ -154,10 +181,8 @@ static std::unique_ptr BackwardRecursive( net->InsertOp(pos.first + 1, std::move(pos.second)); } } else { - OpDescBind fwd_desc; - fwd_desc.SetInput(forwardOp.Inputs()); - - std::unique_ptr grad_op(OpRegistry::CreateGradOp(forwardOp)); + std::unique_ptr grad_op(CreateGradOp(forwardOp)); + PADDLE_ENFORCE(grad_op != nullptr); ForEachVarName(grad_op->Inputs(), [&no_grad_names, &net, &grad_op]( const std::string& grad_input) { diff --git a/paddle/framework/op_desc.h b/paddle/framework/op_desc.h index ec92d08768..72d7a0379b 100644 --- a/paddle/framework/op_desc.h +++ b/paddle/framework/op_desc.h @@ -76,18 +76,22 @@ class OpDescBind { return MapKeys(outputs_); } - void SetInput( - const std::unordered_map> &input) { + void SetInputMap(const VariableNameMap &input) { this->inputs_ = input; this->need_update_ = true; } - void SetOutput( - const std::unordered_map> &output) { + void SetOutputMap(const VariableNameMap &output) { this->outputs_ = output; this->need_update_ = true; } + void Sync(); + + const VariableNameMap &Inputs() const { return inputs_; } + + const VariableNameMap &Outputs() const { return outputs_; } + private: template static std::vector MapKeys(const MapType &map) { @@ -99,8 +103,6 @@ class OpDescBind { return ret_val; } - void Sync(); - OpDesc op_desc_; VariableNameMap inputs_; VariableNameMap outputs_; diff --git a/paddle/framework/op_registry.cc b/paddle/framework/op_registry.cc index 0a2b6fd582..35f280981b 100644 --- a/paddle/framework/op_registry.cc +++ b/paddle/framework/op_registry.cc @@ -52,5 +52,11 @@ std::unique_ptr OpRegistry::CreateOp(const OpDesc& op_desc) { return CreateOp(op_desc.type(), inputs, outputs, attrs); } +std::unique_ptr OpRegistry::CreateOp(OpDescBind* op_desc) { + op_desc->Sync(); + return CreateOp(op_desc->Type(), op_desc->Inputs(), op_desc->Outputs(), + op_desc->GetAttrMap()); +} + } // namespace framework } // namespace paddle diff --git a/paddle/framework/op_registry.h b/paddle/framework/op_registry.h index 0f377f34cb..d14f70008b 100644 --- a/paddle/framework/op_registry.h +++ b/paddle/framework/op_registry.h @@ -23,8 +23,8 @@ limitations under the License. */ #include "paddle/framework/attribute.h" #include "paddle/framework/details/op_registry.h" #include "paddle/framework/framework.pb.h" -#include "paddle/framework/grad_op_builder.h" #include "paddle/framework/grad_op_desc_maker.h" +#include "paddle/framework/op_desc.h" #include "paddle/framework/operator.h" #include "paddle/framework/scope.h" @@ -46,15 +46,15 @@ class Registrar { template struct OperatorRegistrar : public Registrar { explicit OperatorRegistrar(const char* op_type) : op_type(op_type) { + std::cerr << "Reg operator " << op_type << std::endl; PADDLE_ENFORCE(!OpInfoMap::Instance().Has(op_type), "'%s' is registered more than once.", op_type); static_assert(sizeof...(ARGS) != 0, "OperatorRegistrar should be invoked at least by OpClass"); details::OperatorRegistrarRecursive<0, false, ARGS...>(op_type, &info); + OpInfoMap::Instance().Insert(op_type, info); } - ~OperatorRegistrar() { OpInfoMap::Instance().Insert(op_type, info); } - const char* op_type; OpInfo info; @@ -79,6 +79,8 @@ class OpRegistry { AttributeMap attrs); static std::unique_ptr CreateOp(const OpDesc& op_desc); + + static std::unique_ptr CreateOp(OpDescBind* op_desc); }; template From ff8766e910a4d9ba1e208458de2719708d6663d3 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Mon, 2 Oct 2017 15:50:00 -0700 Subject: [PATCH 097/170] Stash --- paddle/framework/backward_test.cc | 2 ++ paddle/framework/details/op_registry.h | 1 + paddle/framework/op_desc.h | 2 +- paddle/framework/op_info.h | 8 -------- paddle/framework/op_proto_maker.h | 5 ----- paddle/framework/op_registry.cc | 4 +++- 6 files changed, 7 insertions(+), 15 deletions(-) diff --git a/paddle/framework/backward_test.cc b/paddle/framework/backward_test.cc index 28fc6f9ced..85f1dd91ed 100644 --- a/paddle/framework/backward_test.cc +++ b/paddle/framework/backward_test.cc @@ -378,6 +378,8 @@ TEST(Backward, linear_net_intermediate_variable_has_no_grad) { + 1UL /* external output number*/ + 1UL /* number of gradient of external output*/ + 2U /* internal variable number*/); + std::cerr << grad_fc.DebugString() << std::endl; + EXPECT_EQ(grad_fc.Outputs(all).size(), 2UL /* input number of mul*/ + 2UL /* input number of rowwise_add diff --git a/paddle/framework/details/op_registry.h b/paddle/framework/details/op_registry.h index daa474e8c5..c805dae7d7 100644 --- a/paddle/framework/details/op_registry.h +++ b/paddle/framework/details/op_registry.h @@ -85,6 +85,7 @@ struct OpInfoFiller { info->proto_ = new OpProto; info->checker_ = new OpAttrChecker(); auto maker = T(info->proto_, info->checker_); + std::cerr << "Assign Maker " << op_type << std::endl; maker.Validate(); info->proto_->set_type(op_type); PADDLE_ENFORCE( diff --git a/paddle/framework/op_desc.h b/paddle/framework/op_desc.h index 72d7a0379b..4c1ada05f0 100644 --- a/paddle/framework/op_desc.h +++ b/paddle/framework/op_desc.h @@ -98,7 +98,7 @@ class OpDescBind { std::vector ret_val; ret_val.reserve(map.size()); std::transform( - map.begin(), map.end(), ret_val.begin(), + map.begin(), map.end(), std::back_inserter(ret_val), [](const typename MapType::value_type &pair) { return pair.first; }); return ret_val; } diff --git a/paddle/framework/op_info.h b/paddle/framework/op_info.h index 683476dfd4..ab13dad962 100644 --- a/paddle/framework/op_info.h +++ b/paddle/framework/op_info.h @@ -42,19 +42,11 @@ struct OpInfo { return *proto_; } - const OpAttrChecker& Checker() const { - PADDLE_ENFORCE_NOT_NULL(checker_, - "Operator Checker has not been registered"); - return *checker_; - } - const OpCreator& Creator() const { PADDLE_ENFORCE_NOT_NULL(creator_, "Operator Creator has not been registered"); return creator_; } - - bool HasGradientOp() const { return !grad_op_type_.empty(); } }; class OpInfoMap { diff --git a/paddle/framework/op_proto_maker.h b/paddle/framework/op_proto_maker.h index 4d55a37db9..a134befd90 100644 --- a/paddle/framework/op_proto_maker.h +++ b/paddle/framework/op_proto_maker.h @@ -44,11 +44,6 @@ class OpProtoAndCheckerMaker { var_->set_intermediate(true); return *this; } - - VariableBuilder& NotInGradient() { - var_->set_not_in_gradient(true); - return *this; - } }; VariableBuilder AddInput(const std::string& name, const std::string& comment); diff --git a/paddle/framework/op_registry.cc b/paddle/framework/op_registry.cc index 35f280981b..ac6aa8d28e 100644 --- a/paddle/framework/op_registry.cc +++ b/paddle/framework/op_registry.cc @@ -23,7 +23,9 @@ std::unique_ptr OpRegistry::CreateOp( const std::string& type, const VariableNameMap& inputs, const VariableNameMap& outputs, AttributeMap attrs) { auto& info = OpInfoMap::Instance().Get(type); - info.Checker().Check(attrs); + if (info.checker_ != nullptr) { + info.checker_->Check(attrs); + } auto op = info.Creator()(type, inputs, outputs, attrs); return std::unique_ptr(op); } From 84b8baf1967e327712269e7632235438d09759d9 Mon Sep 17 00:00:00 2001 From: zchen0211 Date: Mon, 2 Oct 2017 15:50:24 -0700 Subject: [PATCH 098/170] gather scatter with cuda streams --- paddle/operators/gather.cu.h | 13 ++++++++----- paddle/operators/gather_op.cu | 5 ++--- paddle/operators/scatter.cu.h | 10 ++++++---- paddle/operators/scatter_op.cu | 4 ++-- 4 files changed, 18 insertions(+), 14 deletions(-) diff --git a/paddle/operators/gather.cu.h b/paddle/operators/gather.cu.h index b400c10440..2ae11376a2 100644 --- a/paddle/operators/gather.cu.h +++ b/paddle/operators/gather.cu.h @@ -46,9 +46,9 @@ __global__ void GatherCUDAKernel(const T* params, const int* indices, T* output, * return: output tensor */ template -void GPUGather(const Place& place, const Tensor* src, const Tensor* index, - Tensor* output) { - PADDLE_ENFORCE(platform::is_gpu_place(place)); +void GPUGather(const platform::DeviceContext& ctx, const Tensor* src, + const Tensor* index, Tensor* output) { + // PADDLE_ENFORCE(platform::is_gpu_place(place)); // check index of shape 1-D PADDLE_ENFORCE(index->dims().size() == 1); int index_size = index->dims()[0]; @@ -68,8 +68,11 @@ void GPUGather(const Place& place, const Tensor* src, const Tensor* index, int block = 512; int n = slice_size * index_size; int grid = (n + block - 1) / block; - GatherCUDAKernel<<>>(p_src, p_index, p_output, index_size, - slice_size); + + GatherCUDAKernel<<< + grid, block, 0, + reinterpret_cast(ctx).stream()>>>( + p_src, p_index, p_output, index_size, slice_size); } } // namespace operators diff --git a/paddle/operators/gather_op.cu b/paddle/operators/gather_op.cu index 06004614b2..9937be5915 100644 --- a/paddle/operators/gather_op.cu +++ b/paddle/operators/gather_op.cu @@ -32,7 +32,7 @@ class GatherOpCUDAKernel : public framework::OpKernel { output->mutable_data(ctx.GetPlace()); - GPUGather(ctx.GetPlace(), x, index, output); + GPUGather(ctx.device_context(), x, index, output); } }; @@ -42,7 +42,6 @@ class GatherGradOpCUDAKernel : public framework::OpKernel { void Compute(const framework::ExecutionContext &ctx) const override { PADDLE_ENFORCE(platform::is_gpu_place(ctx.GetPlace()), "This kernel only runs on GPU device."); - LOG(INFO) << "Gather grad here"; auto *Index = ctx.Input("Index"); auto *dX = ctx.Output(framework::GradVarName("X")); auto *dO = ctx.Input(framework::GradVarName("Out")); @@ -53,7 +52,7 @@ class GatherGradOpCUDAKernel : public framework::OpKernel { auto place = ctx.GetEigenDevice(); dxt.device(place) = dxt.constant(static_cast(0)); - GPUScatterAssign(ctx.GetPlace(), dO, Index, dX); + GPUScatterAssign(ctx.device_context(), dO, Index, dX); } }; diff --git a/paddle/operators/scatter.cu.h b/paddle/operators/scatter.cu.h index add4791a79..f4a3965d94 100644 --- a/paddle/operators/scatter.cu.h +++ b/paddle/operators/scatter.cu.h @@ -45,11 +45,11 @@ __global__ void ScatterCUDAKernel(const T* params, const int* indices, * return: output tensor */ template -void GPUScatterAssign(const platform::Place& place, +void GPUScatterAssign(const platform::DeviceContext& ctx, const paddle::framework::Tensor* src, const paddle::framework::Tensor* index, paddle::framework::Tensor* output) { - PADDLE_ENFORCE(platform::is_gpu_place(place)); + // PADDLE_ENFORCE(platform::is_gpu_place(place)); // check index of shape 1-D PADDLE_ENFORCE(index->dims().size() == 1); int index_size = index->dims()[0]; @@ -70,8 +70,10 @@ void GPUScatterAssign(const platform::Place& place, int n = slice_size * index_size; int grid = (n + block - 1) / block; - ScatterCUDAKernel<<>>(p_src, p_index, p_output, index_size, - slice_size); + ScatterCUDAKernel<<< + grid, block, 0, + reinterpret_cast(ctx).stream()>>>( + p_src, p_index, p_output, index_size, slice_size); } } // namespace operators diff --git a/paddle/operators/scatter_op.cu b/paddle/operators/scatter_op.cu index 831eabdae4..6d13a876f9 100644 --- a/paddle/operators/scatter_op.cu +++ b/paddle/operators/scatter_op.cu @@ -32,7 +32,7 @@ class ScatterOpCUDAKernel : public framework::OpKernel { Out->ShareDataWith(*Ref); - GPUScatterAssign(ctx.GetPlace(), Updates, Index, Out); + GPUScatterAssign(ctx.device_context(), Updates, Index, Out); } }; @@ -51,7 +51,7 @@ class ScatterGradOpCUDAKernel : public framework::OpKernel { dRef->ShareDataWith(*dOut); dUpdates->mutable_data(ctx.GetPlace()); // Gradient by Gather: dUpdates = dO[Index] - GPUGather(ctx.GetPlace(), dOut, Index, dUpdates); + GPUGather(ctx.device_context(), dOut, Index, dUpdates); } }; From 9ff1fd41b2e8769d233e160975e036f539cda99f Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Mon, 2 Oct 2017 15:57:18 -0700 Subject: [PATCH 099/170] Fix MacOS compile error The private data `tensor_shared_` is not used. --- paddle/framework/tensor_array.h | 8 -------- 1 file changed, 8 deletions(-) diff --git a/paddle/framework/tensor_array.h b/paddle/framework/tensor_array.h index e76f33d2c0..22ae6a966f 100644 --- a/paddle/framework/tensor_array.h +++ b/paddle/framework/tensor_array.h @@ -47,13 +47,6 @@ class TensorArray { // max number of values allowed to store. const size_t MAX_SIZE{100000}; - /* - * Inputs: - * - value_shared: share memory between tensors. - */ - explicit TensorArray(bool values_shared = true) - : values_shared_(values_shared) {} - /* * Read the value at location `index` in the `TensorArray`. */ @@ -111,7 +104,6 @@ class TensorArray { private: mutable std::vector values_; - bool values_shared_; }; // class TensorArray } // namespace framework From adec0d30fe8454f84b6bc61cc8b0385f6483d0c3 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Mon, 2 Oct 2017 16:18:26 -0700 Subject: [PATCH 100/170] Simplify SumOp Kernel --- paddle/operators/CMakeLists.txt | 6 +++++- paddle/operators/sum_op.cc | 29 +++++++++++++++-------------- paddle/operators/sum_op.cu | 4 +--- paddle/operators/sum_op.h | 19 ------------------- 4 files changed, 21 insertions(+), 37 deletions(-) diff --git a/paddle/operators/CMakeLists.txt b/paddle/operators/CMakeLists.txt index 43eb4de2c1..0fa1fca2bc 100644 --- a/paddle/operators/CMakeLists.txt +++ b/paddle/operators/CMakeLists.txt @@ -103,12 +103,16 @@ set(DEPS_OPS recurrent_op cond_op cross_entropy_op - softmax_with_cross_entropy_op) + softmax_with_cross_entropy_op + sum_op) + + op_library(recurrent_op SRCS recurrent_op.cc rnn/recurrent_op_utils.cc DEPS framework_proto tensor net_op) op_library(cond_op SRCS cond_op.cc DEPS framework_proto tensor operator net_op) op_library(cross_entropy_op DEPS cross_entropy) op_library(softmax_with_cross_entropy_op DEPS cross_entropy softmax) +op_library(sum_op DEPS net_op) list(REMOVE_ITEM GENERAL_OPS ${DEPS_OPS}) foreach(src ${GENERAL_OPS}) diff --git a/paddle/operators/sum_op.cc b/paddle/operators/sum_op.cc index c54843faa6..7c422b4770 100644 --- a/paddle/operators/sum_op.cc +++ b/paddle/operators/sum_op.cc @@ -11,6 +11,7 @@ limitations under the License. */ #include "paddle/operators/sum_op.h" #include +#include "paddle/operators/net_op.h" namespace paddle { namespace operators { @@ -57,21 +58,23 @@ or not. But the output only shares the LoD with the first input. } }; -class SumGradOp : public framework::OperatorWithKernel { +class SumGradOp : public NetOp { public: - using framework::OperatorWithKernel::OperatorWithKernel; + SumGradOp(const std::string& type, const framework::VariableNameMap& inputs, + const framework::VariableNameMap& outputs, + const framework::AttributeMap& attrs) + : NetOp(type, inputs, outputs, attrs) { + auto& x_grad_names = Outputs(framework::GradVarName("X")); + auto out_grad_name = this->Input(framework::GradVarName("Out")); - protected: - void InferShape(framework::InferShapeContextBase* ctx) const override { - auto out_grad_dims = ctx->GetInputDim(framework::GradVarName("Out")); - auto x_grad_names = ctx->Outputs(framework::GradVarName("X")); - size_t x_length = x_grad_names.size(); - std::vector x_grad_dims; - x_grad_dims.reserve(x_length); - for (size_t i = 0; i < x_length; ++i) { - x_grad_dims.push_back(out_grad_dims); + framework::AttributeMap grad_attrs; + grad_attrs["scale"] = 1.0f; + for (auto& x_grad_name : x_grad_names) { + AppendOp(framework::OpRegistry::CreateOp( + "scale", {{"X", {out_grad_name}}}, {{"Out", {x_grad_name}}}, + grad_attrs)); } - ctx->SetOutputsDim(framework::GradVarName("X"), x_grad_dims); + CompleteAddOp(false); } }; @@ -81,5 +84,3 @@ class SumGradOp : public framework::OperatorWithKernel { namespace ops = paddle::operators; REGISTER_OP(sum, ops::SumOp, ops::SumOpMaker, sum_grad, ops::SumGradOp); REGISTER_OP_CPU_KERNEL(sum, ops::SumKernel); -REGISTER_OP_CPU_KERNEL(sum_grad, - ops::SumGradKernel); diff --git a/paddle/operators/sum_op.cu b/paddle/operators/sum_op.cu index a465cf3659..7129e6bf62 100644 --- a/paddle/operators/sum_op.cu +++ b/paddle/operators/sum_op.cu @@ -13,6 +13,4 @@ limitations under the License. */ #include "paddle/operators/sum_op.h" namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL(sum, ops::SumKernel); -REGISTER_OP_GPU_KERNEL(sum_grad, - ops::SumGradKernel); +REGISTER_OP_GPU_KERNEL(sum, ops::SumKernel); \ No newline at end of file diff --git a/paddle/operators/sum_op.h b/paddle/operators/sum_op.h index 7e8fbb9e41..91e5da8b40 100644 --- a/paddle/operators/sum_op.h +++ b/paddle/operators/sum_op.h @@ -42,24 +42,5 @@ class SumKernel : public framework::OpKernel { } }; -template -class SumGradKernel : public framework::OpKernel { - public: - void Compute(const framework::ExecutionContext& context) const override { - auto* input = context.Input(framework::GradVarName("Out")); - auto outs = context.MultiOutput(framework::GradVarName("X")); - for (auto out : outs) { - out->mutable_data(context.GetPlace()); - } - - auto place = context.GetEigenDevice(); - auto in = EigenVector::Flatten(*input); - for (auto out : outs) { - auto result = EigenVector::Flatten(*out); - result.device(place) = in; - } - } -}; - } // namespace operators } // namespace paddle From d550380ed92ed9c762ff0248780a3c28dbf27416 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Mon, 2 Oct 2017 16:52:11 -0700 Subject: [PATCH 101/170] add CompileTimeInferShapeContext --- paddle/framework/CMakeLists.txt | 2 +- paddle/framework/block_desc.cc | 5 ++ paddle/framework/block_desc.h | 2 + paddle/framework/operator.h | 84 +++++++++++++-------------------- 4 files changed, 41 insertions(+), 52 deletions(-) diff --git a/paddle/framework/CMakeLists.txt b/paddle/framework/CMakeLists.txt index 5d394132b7..a2efcdb55c 100644 --- a/paddle/framework/CMakeLists.txt +++ b/paddle/framework/CMakeLists.txt @@ -23,7 +23,7 @@ cc_library(proto_desc SRCS var_desc.cc op_desc.cc block_desc.cc program_desc.cc cc_library(op_proto_maker SRCS op_proto_maker.cc DEPS framework_proto attribute) cc_test(op_proto_maker_test SRCS op_proto_maker_test.cc DEPS op_proto_maker) cc_library(op_info SRCS op_info.cc DEPS attribute framework_proto proto_desc) -cc_library(operator SRCS operator.cc DEPS op_info device_context tensor scope) +cc_library(operator SRCS operator.cc DEPS op_info device_context tensor scope proto_desc) cc_test(operator_test SRCS operator_test.cc DEPS operator op_registry) cc_library(grad_op_builder SRCS grad_op_builder.cc DEPS operator proto_desc) diff --git a/paddle/framework/block_desc.cc b/paddle/framework/block_desc.cc index 9570aedfdd..670533a3fe 100644 --- a/paddle/framework/block_desc.cc +++ b/paddle/framework/block_desc.cc @@ -34,6 +34,11 @@ VarDescBind *BlockDescBind::Var(const std::string &name) const { return it->second.get(); } +bool BlockDescBind::HasVar(const std::string &name) const { + auto it = vars_.find(name); + return it != vars_.end(); +} + std::vector BlockDescBind::AllVars() const { std::vector res; for (const auto &p : vars_) { diff --git a/paddle/framework/block_desc.h b/paddle/framework/block_desc.h index 1a1135bab4..41cf1dc385 100644 --- a/paddle/framework/block_desc.h +++ b/paddle/framework/block_desc.h @@ -45,6 +45,8 @@ class BlockDescBind { VarDescBind *Var(const std::string &name_bytes) const; + bool HasVar(const std::string &var_name) const; + std::vector AllVars() const; BlockDescBind *ParentBlock() const; diff --git a/paddle/framework/operator.h b/paddle/framework/operator.h index f807909650..2874237b9c 100644 --- a/paddle/framework/operator.h +++ b/paddle/framework/operator.h @@ -319,100 +319,82 @@ class ExecutionContext : public InferShapeContext { class CompileTimeInferShapeContext : public InferShapeContextBase { public: - CompileTimeInferShapeContext(const OperatorBase& op, const Scope& scope) - : op_(op), scope_(scope) {} + CompileTimeInferShapeContext(const OpDescBind& op, const BlockDescBind& block) + : op_(op), block_(block) {} bool HasInput(const std::string& name) const { - auto ipt = op_.Input(name); - auto* var = ipt == kEmptyVarName ? nullptr : scope_.FindVar(ipt); - return var != nullptr; + const std::vector& input_names = op_.Input(name); + PADDLE_ENFORCE_EQ(input_names.size(), 1UL, "Inputs(%s) length is not 1", + name); + return block_.HasVar(input_names[0]); } bool HasOutput(const std::string& name) const { - auto ipt = op_.Output(name); - auto* var = ipt == kEmptyVarName ? nullptr : scope_.FindVar(ipt); - return var != nullptr; + const std::vector& output_names = op_.Output(name); + PADDLE_ENFORCE_EQ(output_names.size(), 1UL, "Outputs(%s) length is not 1", + name); + return block_.HasVar(output_names[0]); } bool HasInputs(const std::string& name) const { - auto inputs = op_.Inputs(name); - if (inputs.size() == 0UL) { - return false; - } - for (auto& input : inputs) { - if (scope_.FindVar(input) == nullptr) { - return false; - } + const std::vector& input_names = op_.Input(name); + PADDLE_ENFORCE_GT(input_names.size(), 0UL, "Inputs(%s) length is 0", name); + for (auto& input : input_names) { + if (!block_.HasVar(input)) return false; } return true; } bool HasOutputs(const std::string& name) const { - auto outputs = op_.Outputs(name); - if (outputs.size() == 0UL) { - return false; - } - for (auto& output : outputs) { - if (scope_.FindVar(output) == nullptr) { - return false; - } + const std::vector& output_names = op_.Output(name); + PADDLE_ENFORCE_GT(output_names.size(), 0UL, "Inputs(%s) length is 0", name); + for (auto& output : output_names) { + if (!block_.HasVar(name)) return false; } return true; } DDim GetInputDim(const std::string& name) const { - return GetDim(op_.Input(name)); + std::vector ddims = GetInputsDim(name); + PADDLE_ENFORCE_EQ(ddims.size(), 1UL, "Inputs(%s) length is not 1", name); + return ddims[0]; } void SetInputDim(const std::string& name, const DDim& dim) { - SetDim(op_.Input(name), dim); + SetInputsDim(name, {dim}); } DDim GetOutputDim(const std::string& name) const { - return GetDim(op_.Output(name)); + std::vector ddims = GetOutputsDim(name); + PADDLE_ENFORCE_EQ(ddims.size(), 1UL, "Outputs(%s) length is not 1", name); + return ddims[0]; } void SetOutputDim(const std::string& name, const DDim& dim) { - SetDim(op_.Output(name), dim); + SetOutputsDim(name, {dim}); } - AttrReader Attrs() const { return AttrReader(op_.Attrs()); } + AttrReader Attrs() const { return AttrReader(op_.GetAttrMap()); } const std::vector& Inputs(const std::string& name) const { - return op_.Inputs(name); + return op_.Input(name); } const std::vector& Outputs(const std::string& name) const { - return op_.Outputs(name); + return op_.Output(name); } private: - template - Tensor* GetTensor(const std::string& name) const { - Tensor* t = nullptr; - auto* var = scope_.FindVar(name); - if (!var->IsType() && !var->IsType()) { - if (Allocate) { - t = var->GetMutable(); - } else { - PADDLE_THROW("Variable(%s) should be tensor", name); - } - } else { - t = GetTensorFromVar(scope_.FindVar(name)); - } - return t; - } - DDim GetDim(const std::string& name) const { - return GetTensor(name)->dims(); + return framework::make_ddim(block_.Var(name)->Shape()); } void SetDim(const std::string& name, const DDim& dim) { - GetTensor(name)->Resize(dim); + block_.Var(name)->SetShape(framework::vectorize(dim)); } - const OperatorBase& op_; - const Scope& scope_; + const OpDescBind& op_; + const BlockDescBind& block_; }; class RuntimeInferShapeContext : public InferShapeContextBase { From ff1bfdedc97eb0834745e812abd619581fde7950 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Mon, 2 Oct 2017 17:09:25 -0700 Subject: [PATCH 102/170] Fix CRLF in sum_op.cu --- paddle/operators/sum_op.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/operators/sum_op.cu b/paddle/operators/sum_op.cu index 7129e6bf62..b1896d3cd8 100644 --- a/paddle/operators/sum_op.cu +++ b/paddle/operators/sum_op.cu @@ -13,4 +13,4 @@ limitations under the License. */ #include "paddle/operators/sum_op.h" namespace ops = paddle::operators; -REGISTER_OP_GPU_KERNEL(sum, ops::SumKernel); \ No newline at end of file +REGISTER_OP_GPU_KERNEL(sum, ops::SumKernel); From 61c03f9d59969d698faca1547115d80e8faa3e9d Mon Sep 17 00:00:00 2001 From: Kavya Srinet Date: Mon, 2 Oct 2017 17:52:56 -0700 Subject: [PATCH 103/170] Adding the implementation for rmsprop operator --- paddle/operators/rmsprop_op.cc | 87 +++++++++++++++++++ paddle/operators/rmsprop_op.cu | 20 +++++ paddle/operators/rmsprop_op.h | 54 ++++++++++++ .../v2/framework/tests/test_rmsprop_op.py | 37 ++++++++ 4 files changed, 198 insertions(+) create mode 100644 paddle/operators/rmsprop_op.cc create mode 100644 paddle/operators/rmsprop_op.cu create mode 100644 paddle/operators/rmsprop_op.h create mode 100644 python/paddle/v2/framework/tests/test_rmsprop_op.py diff --git a/paddle/operators/rmsprop_op.cc b/paddle/operators/rmsprop_op.cc new file mode 100644 index 0000000000..dcf3599f4d --- /dev/null +++ b/paddle/operators/rmsprop_op.cc @@ -0,0 +1,87 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/operators/rmsprop_op.h" + +namespace paddle { +namespace operators { + +class RmspropOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(framework::InferShapeContextBase *ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("Param"), + "Input(param) of RmspropOp should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Grad"), + "Input(grad) of RmspropOp should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Moment"), + "Input(moment) of RmspropOp should not be null."); + + PADDLE_ENFORCE(ctx->HasOutput("ParamOut"), + "Output(param_out) of RmspropOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("MomentOut"), + "Output(moment_out) of RmspropOp should not be null."); + + auto param_dim = ctx->GetInputDim("Param"); + PADDLE_ENFORCE_EQ( + param_dim, ctx->GetInputDim("Grad"), + "Param and grad input of RmspropOp should have the same dimension."); + PADDLE_ENFORCE_EQ( + param_dim, ctx->GetInputDim("Moment"), + "Param and moment input of RmspropOp should have the same dimension."); + + ctx->SetOutputDim("ParamOut", param_dim); + ctx->SetOutputDim("MomentOut", param_dim); + } +}; + +class RmspropOpMaker : public framework::OpProtoAndCheckerMaker { + public: + RmspropOpMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("Param", "Input parameter"); + AddInput("Grad", "Input gradient"); + AddInput("Moment", "Second moment"); + + AddOutput("ParamOut", "Output parameter"); + AddOutput("MomentOut", "Output second moment"); + + AddAttr("learningRate", "Learning rate"); + AddAttr("epsilon", "Constant for numerical stability"); + AddAttr("decayRate", "Decay rate for moving average of gradients"); + AddComment(R"DOC( + +RMSprop + +MomentOut = decayRate * Moment + (1 - decayRate) * Grad * Grad +ParamOut = Param - learningRate * Grad / (sqrt(MomentOut) + epsilon) + +The original slide(Slide 29 of +http://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf) +does not have the epsilon attribute. It is added here for numerical stability +to avoid division by zero. + +)DOC"); + } +}; +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP_WITHOUT_GRADIENT(rmsprop, ops::RmspropOp, ops::RmspropOpMaker); +REGISTER_OP_CPU_KERNEL(rmsprop, + ops::RmspropOpKernel); diff --git a/paddle/operators/rmsprop_op.cu b/paddle/operators/rmsprop_op.cu new file mode 100644 index 0000000000..52634a5481 --- /dev/null +++ b/paddle/operators/rmsprop_op.cu @@ -0,0 +1,20 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#define EIGEN_USE_GPU +#include "paddle/operators/rmsprop_op.h" + +namespace ops = paddle::operators; +REGISTER_OP_GPU_KERNEL(rmsprop, + ops::RmspropOpKernel); diff --git a/paddle/operators/rmsprop_op.h b/paddle/operators/rmsprop_op.h new file mode 100644 index 0000000000..c94c24bddd --- /dev/null +++ b/paddle/operators/rmsprop_op.h @@ -0,0 +1,54 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once +#include "paddle/framework/eigen.h" +#include "paddle/framework/op_registry.h" + +namespace paddle { +namespace operators { + +using Tensor = framework::Tensor; +template +using EigenVector = framework::EigenVector; + +template +class RmspropOpKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const override { + auto param_out = ctx.Output("ParamOut"); + auto moment_out = ctx.Output("MomentOut"); + + param_out->mutable_data(ctx.GetPlace()); + moment_out->mutable_data(ctx.GetPlace()); + + float lr = ctx.Attr("learningRate"); + float epsilon = ctx.Attr("epsilon"); + float decay = ctx.Attr("decayRate"); + + auto p = EigenVector::Flatten(*ctx.Input("Param")); + auto g = EigenVector::Flatten(*ctx.Input("Grad")); + auto m = EigenVector::Flatten(*ctx.Input("Moment")); + auto p_out = EigenVector::Flatten(*param_out); + auto m_out = EigenVector::Flatten(*moment_out); + auto place = ctx.GetEigenDevice(); + + m_out.device(place) = decay * m + (1 - decay) * g * g; + p_out.device(place) = p - lr * g / (m_out.sqrt() + epsilon); + } +}; + +} // namespace operators +} // namespace paddle diff --git a/python/paddle/v2/framework/tests/test_rmsprop_op.py b/python/paddle/v2/framework/tests/test_rmsprop_op.py new file mode 100644 index 0000000000..1fc59a0f11 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_rmsprop_op.py @@ -0,0 +1,37 @@ +import unittest +import numpy as np +from op_test import OpTest + + +class TestRmspropOp(OpTest): + def setUp(self): + self.op_type = "rmsprop" + + param = np.random.random((123, 321)).astype("float32") + grad = np.random.random((123, 321)).astype("float32") + moment = np.zeros((123, 321)).astype("float32") + + learning_rate = 0.01 + epsilon = 1e-6 + decay_rate = 0.9 + + self.inputs = {'Param': param, 'Grad': grad, 'Moment': moment} + + self.attrs = { + 'learningRate': learning_rate, + 'epsilon': epsilon, + 'decayRate': decay_rate + } + + moment_out = decay_rate * moment + (1 - decay_rate) * grad * grad + param_out = param - learning_rate * grad / (np.sqrt(moment_out) + + epsilon) + + self.outputs = {'ParamOut': param_out, 'MomentOut': moment_out} + + def test_check_output(self): + self.check_output() + + +if __name__ == "__main__": + unittest.main() From 455436e5148ad0a84cae89e46931e4785c57870d Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Mon, 2 Oct 2017 18:04:00 -0700 Subject: [PATCH 104/170] fix compile problem --- paddle/framework/operator.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/paddle/framework/operator.h b/paddle/framework/operator.h index 2874237b9c..5bb5c8e2f9 100644 --- a/paddle/framework/operator.h +++ b/paddle/framework/operator.h @@ -22,6 +22,7 @@ limitations under the License. */ #include "op_info.h" #include "paddle/framework/attribute.h" +#include "paddle/framework/block_desc.h" #include "paddle/framework/data_type.h" #include "paddle/framework/framework.pb.h" #include "paddle/framework/lod_tensor.h" @@ -349,7 +350,7 @@ class CompileTimeInferShapeContext : public InferShapeContextBase { const std::vector& output_names = op_.Output(name); PADDLE_ENFORCE_GT(output_names.size(), 0UL, "Inputs(%s) length is 0", name); for (auto& output : output_names) { - if (!block_.HasVar(name)) return false; + if (!block_.HasVar(output)) return false; } return true; } From 46c551b2997537a70ea82fd55067fd57cc4c59d5 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Mon, 2 Oct 2017 18:27:21 -0700 Subject: [PATCH 105/170] Complete Register Gradient in compile time --- paddle/framework/backward_test.cc | 32 ++++++++----- paddle/framework/details/op_registry.h | 1 - paddle/framework/framework.proto | 1 - paddle/framework/op_info.h | 3 ++ paddle/framework/op_registry.h | 1 - paddle/operators/mean_op.cc | 21 ++++++++- paddle/operators/minus_op.cc | 46 +++++++++---------- paddle/operators/pad_op.cc | 22 +++++++-- paddle/operators/scale_op.cc | 33 ++++++------- .../softmax_with_cross_entropy_op.cc | 45 ++++++++++++------ paddle/operators/sum_op.cc | 41 +++++++++-------- 11 files changed, 152 insertions(+), 94 deletions(-) diff --git a/paddle/framework/backward_test.cc b/paddle/framework/backward_test.cc index 85f1dd91ed..93688c383b 100644 --- a/paddle/framework/backward_test.cc +++ b/paddle/framework/backward_test.cc @@ -21,24 +21,34 @@ namespace paddle { namespace framework { -using OperatorBase = framework::OperatorBase; -using OpProtoAndCheckerMaker = framework::OpProtoAndCheckerMaker; -using OpProto = framework::OpProto; -using OpAttrChecker = framework::OpAttrChecker; -using Scope = framework::Scope; using DeviceContext = platform::DeviceContext; class RowWiseAddOpMaker : public OpProtoAndCheckerMaker { public: RowWiseAddOpMaker(OpProto *proto, OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("X", "Input X of Add").NotInGradient(); - AddInput("b", "Bias of Add").NotInGradient(); - AddOutput("Out", "Out of Add").NotInGradient(); + AddInput("X", "Input X of Add"); + AddInput("b", "Bias of Add"); + AddOutput("Out", "Out of Add"); AddComment("Add Op"); } }; +class RowWiseAddGradMaker : public SingleGradOpDescMaker { + public: + using SingleGradOpDescMaker::SingleGradOpDescMaker; + + protected: + OpDescBind Apply() const override { + OpDescBind grad_op; + grad_op.SetInput(GradVarName("Out"), OutputGrad("Out")); + grad_op.SetOutput(GradVarName("X"), InputGrad("X")); + grad_op.SetOutput(GradVarName("b"), InputGrad("b")); + grad_op.SetType("rowwise_add_grad"); + return grad_op; + } +}; + class MulOpMaker : public OpProtoAndCheckerMaker { public: MulOpMaker(OpProto *proto, OpAttrChecker *op_checker) @@ -148,8 +158,9 @@ class AddOpMaker : public OpProtoAndCheckerMaker { namespace f = paddle::framework; namespace ops = paddle::operators; using EnforceNotMet = paddle::platform::EnforceNotMet; -REGISTER_OP(rowwise_add, f::NOP, f::RowWiseAddOpMaker, rowwise_add_grad, - f::NOP); +REGISTER_OPERATOR(rowwise_add, f::NOP, f::RowWiseAddOpMaker, + f::RowWiseAddGradMaker); +REGISTER_OPERATOR(rowwise_add_grad, f::NOP); REGISTER_OP(mul, f::NOP, f::MulOpMaker, mul_grad, f::NOP); REGISTER_OP(sigmoid, f::NOP, f::SigmoidOpMaker, sigmoid_grad, f::NOP); REGISTER_OP_WITHOUT_GRADIENT(nograd, f::NOP, f::NoGradOpMaker); @@ -378,7 +389,6 @@ TEST(Backward, linear_net_intermediate_variable_has_no_grad) { + 1UL /* external output number*/ + 1UL /* number of gradient of external output*/ + 2U /* internal variable number*/); - std::cerr << grad_fc.DebugString() << std::endl; EXPECT_EQ(grad_fc.Outputs(all).size(), 2UL /* input number of mul*/ diff --git a/paddle/framework/details/op_registry.h b/paddle/framework/details/op_registry.h index c805dae7d7..daa474e8c5 100644 --- a/paddle/framework/details/op_registry.h +++ b/paddle/framework/details/op_registry.h @@ -85,7 +85,6 @@ struct OpInfoFiller { info->proto_ = new OpProto; info->checker_ = new OpAttrChecker(); auto maker = T(info->proto_, info->checker_); - std::cerr << "Assign Maker " << op_type << std::endl; maker.Validate(); info->proto_->set_type(op_type); PADDLE_ENFORCE( diff --git a/paddle/framework/framework.proto b/paddle/framework/framework.proto index 951c7afbc1..e90a816afa 100644 --- a/paddle/framework/framework.proto +++ b/paddle/framework/framework.proto @@ -66,7 +66,6 @@ message OpProto { optional bool duplicable = 3 [ default = false ]; optional bool intermediate = 4 [ default = false ]; - optional bool not_in_gradient = 5 [ default = false ]; } // AttrProto describes the C++ type Attribute. diff --git a/paddle/framework/op_info.h b/paddle/framework/op_info.h index 2d9568c320..8c2a9178a7 100644 --- a/paddle/framework/op_info.h +++ b/paddle/framework/op_info.h @@ -17,11 +17,14 @@ #include #include #include + #include "paddle/framework/attribute.h" #include "paddle/framework/op_desc.h" #include "paddle/framework/type_defs.h" #include "paddle/platform/macros.h" +#include "glog/logging.h" + namespace paddle { namespace framework { diff --git a/paddle/framework/op_registry.h b/paddle/framework/op_registry.h index d14f70008b..da112fa488 100644 --- a/paddle/framework/op_registry.h +++ b/paddle/framework/op_registry.h @@ -46,7 +46,6 @@ class Registrar { template struct OperatorRegistrar : public Registrar { explicit OperatorRegistrar(const char* op_type) : op_type(op_type) { - std::cerr << "Reg operator " << op_type << std::endl; PADDLE_ENFORCE(!OpInfoMap::Instance().Has(op_type), "'%s' is registered more than once.", op_type); static_assert(sizeof...(ARGS) != 0, diff --git a/paddle/operators/mean_op.cc b/paddle/operators/mean_op.cc index d799239d4e..0c84cbb5a7 100644 --- a/paddle/operators/mean_op.cc +++ b/paddle/operators/mean_op.cc @@ -36,7 +36,7 @@ class MeanOpMaker : public framework::OpProtoAndCheckerMaker { MeanOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("X", "The input of mean op"); - AddOutput("Out", "The output of mean op").NotInGradient(); + AddOutput("Out", "The output of mean op"); AddComment(R"DOC( Mean Operator )DOC"); } @@ -52,11 +52,28 @@ class MeanGradOp : public framework::OperatorWithKernel { } }; +class MeanGradMaker : public framework::SingleGradOpDescMaker { + public: + using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; + + protected: + framework::OpDescBind Apply() const override { + framework::OpDescBind grad_op; + grad_op.SetType("mean_grad"); + grad_op.SetInput("X", Input("X")); + grad_op.SetInput(framework::GradVarName("Out"), OutputGrad("Out")); + grad_op.SetOutput(framework::GradVarName("X"), InputGrad("X")); + return grad_op; + } +}; + } // namespace operators } // namespace paddle namespace ops = paddle::operators; -REGISTER_OP(mean, ops::MeanOp, ops::MeanOpMaker, mean_grad, ops::MeanGradOp); + +REGISTER_OPERATOR(mean, ops::MeanOp, ops::MeanOpMaker, ops::MeanGradMaker); +REGISTER_OPERATOR(mean_grad, ops::MeanGradOp); REGISTER_OP_CPU_KERNEL(mean, ops::MeanKernel); REGISTER_OP_CPU_KERNEL(mean_grad, diff --git a/paddle/operators/minus_op.cc b/paddle/operators/minus_op.cc index ce049d4d7b..1b3ae9a9a6 100644 --- a/paddle/operators/minus_op.cc +++ b/paddle/operators/minus_op.cc @@ -49,9 +49,9 @@ class MinusOpMaker : public framework::OpProtoAndCheckerMaker { public: MinusOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("X", "The left tensor of minus operator.").NotInGradient(); - AddInput("Y", "The right tensor of minus operator.").NotInGradient(); - AddOutput("Out", "The output tensor of minus operator.").NotInGradient(); + AddInput("X", "The left tensor of minus operator."); + AddInput("Y", "The right tensor of minus operator."); + AddOutput("Out", "The output tensor of minus operator."); AddComment(R"DOC(Minus Operator @@ -64,26 +64,25 @@ or not. But the output only shares the LoD with input `X`. )DOC"); } }; -template -class MinusGradOp : public NetOp { + +class MinusGradMaker : public framework::GradOpDescMakerBase { public: - MinusGradOp(const std::string &type, const framework::VariableNameMap &inputs, - const framework::VariableNameMap &outputs, - const framework::AttributeMap &attrs) - : NetOp(type, inputs, outputs, attrs) { - auto out_grad = Input(framework::GradVarName("Out")); - auto x_grad = Output(framework::GradVarName("X")); - auto y_grad = Output(framework::GradVarName("Y")); - - // x_grad = out_grad - AppendOp(framework::OpRegistry::CreateOp("identity", {{"X", {out_grad}}}, - {{"Y", {x_grad}}}, {})); - - framework::AttributeMap scale_attr; - scale_attr["scale"] = static_cast(-1); - AppendOp(framework::OpRegistry::CreateOp("scale", {{"X", {out_grad}}}, - {{"Out", {y_grad}}}, scale_attr)); - CompleteAddOp(false); + using framework::GradOpDescMakerBase::GradOpDescMakerBase; + + std::vector operator()() const override { + std::vector ops; + ops.resize(2); + + ops[0].SetType("scale"); + ops[0].SetInput("X", OutputGrad("Out")); + ops[0].SetOutput("Out", InputGrad("X")); + ops[0].SetAttr("scale", 1.0f); + + ops[1].SetType("scale"); + ops[1].SetInput("X", OutputGrad("Out")); + ops[1].SetOutput("Out", InputGrad("Y")); + ops[1].SetAttr("scale", -1.0f); + return ops; } }; @@ -91,7 +90,6 @@ class MinusGradOp : public NetOp { } // namespace paddle namespace ops = paddle::operators; -REGISTER_OP(minus, ops::MinusOp, ops::MinusOpMaker, minus_grad, - ops::MinusGradOp); +REGISTER_OPERATOR(minus, ops::MinusOp, ops::MinusOpMaker, ops::MinusGradMaker); REGISTER_OP_CPU_KERNEL(minus, ops::MinusKernel); diff --git a/paddle/operators/pad_op.cc b/paddle/operators/pad_op.cc index 04ebb14f6e..4bd25fa46a 100644 --- a/paddle/operators/pad_op.cc +++ b/paddle/operators/pad_op.cc @@ -56,8 +56,7 @@ class PadOpMaker : public framework::OpProtoAndCheckerMaker { "The input should be a k-D tensor(k > 0 and k < 7)"); AddOutput("Out", "The output of pad op." - "A tensor with the same shape as X.") - .NotInGradient(); + "A tensor with the same shape as X."); AddComment(R"DOC( Pad input into output, as specified by paddings and pad_value. The input should be a k-D tensor(k > 0 and k < 7). As an example: @@ -111,11 +110,28 @@ class PadOpGrad : public framework::OperatorWithKernel { } }; +class PadOpGradMaker : public framework::SingleGradOpDescMaker { + protected: + framework::OpDescBind Apply() const override { + framework::OpDescBind bind; + bind.SetInput("X", Input("X")); + bind.SetInput(framework::GradVarName("Out"), OutputGrad("Out")); + bind.SetOutput(framework::GradVarName("X"), InputGrad("X")); + bind.SetAttrMap(Attrs()); + return bind; + } + + public: + using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; +}; + } // namespace operators } // namespace paddle namespace ops = paddle::operators; -REGISTER_OP(pad, ops::PadOp, ops::PadOpMaker, pad_grad, ops::PadOpGrad); + +REGISTER_OPERATOR(pad, ops::PadOp, ops::PadOpMaker, ops::PadOpGradMaker); +REGISTER_OPERATOR(pad_grad, ops::PadOpGrad); REGISTER_OP_CPU_KERNEL(pad, ops::PadKernel); REGISTER_OP_CPU_KERNEL(pad_grad, ops::PadGradKernel); diff --git a/paddle/operators/scale_op.cc b/paddle/operators/scale_op.cc index e92501e128..40f0960923 100644 --- a/paddle/operators/scale_op.cc +++ b/paddle/operators/scale_op.cc @@ -41,8 +41,8 @@ class ScaleOpMaker : public framework::OpProtoAndCheckerMaker { public: ScaleOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("X", "The input tensor of scale operator.").NotInGradient(); - AddOutput("Out", "The output tensor of scale operator.").NotInGradient(); + AddInput("X", "The input tensor of scale operator."); + AddOutput("Out", "The output tensor of scale operator."); AddComment(R"DOC(Scale operator The equation is: Out = scale*X @@ -52,21 +52,18 @@ The equation is: Out = scale*X } }; -// The operator to calculate gradients of a scale operator is just the scale -// operator itself. -// Grad(Out=scale(X)) => Grad(X) = scale(Grad(Out)) -template -class ScaleGradOp : public NetOp { +class ScaleGradMaker : public framework::SingleGradOpDescMaker { public: - ScaleGradOp(const std::string &type, const framework::VariableNameMap &inputs, - const framework::VariableNameMap &outputs, - const framework::AttributeMap &attrs) - : NetOp(type, inputs, outputs, attrs) { - AppendOp(framework::OpRegistry::CreateOp( - "scale", {{"X", {Input(framework::GradVarName("Out"))}}}, - {{"Out", {Output(framework::GradVarName("X"))}}}, - {{"scale", Attr("scale")}})); - CompleteAddOp(false); + using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; + + protected: + framework::OpDescBind Apply() const override { + framework::OpDescBind grad_op; + grad_op.SetType("scale"); + grad_op.SetInput("X", OutputGrad("Out")); + grad_op.SetOutput("Out", InputGrad("X")); + grad_op.SetAttr("scale", GetAttr("scale")); + return grad_op; } }; @@ -75,7 +72,7 @@ class ScaleGradOp : public NetOp { namespace ops = paddle::operators; -REGISTER_OP(scale, ops::ScaleOp, ops::ScaleOpMaker, scale_grad, - ops::ScaleGradOp); +REGISTER_OPERATOR(scale, ops::ScaleOp, ops::ScaleOpMaker, + ops::ScaleGradMaker); REGISTER_OP_CPU_KERNEL(scale, ops::ScaleKernel); diff --git a/paddle/operators/softmax_with_cross_entropy_op.cc b/paddle/operators/softmax_with_cross_entropy_op.cc index a76489871f..87dcc3f240 100644 --- a/paddle/operators/softmax_with_cross_entropy_op.cc +++ b/paddle/operators/softmax_with_cross_entropy_op.cc @@ -27,15 +27,14 @@ class SoftmaxWithCrossEntropyOpMaker AddInput("Logits", "(Tensor, default: Tensor), The unscaled log probabilities " "which is a 2-D tensor with shape [N x K]. N is the batch_size, " - "and K is the class number.") - .NotInGradient(); - AddInput( - "Label", - "(Tensor, default: Tensor), The ground truth which is a 2-D " - "tensor. " - "If softLable is set to 0, Label is a Tensor with shape [N x 1]. " - "If softLable is set to 1, Label is a Tensor " - "with shape [N x K]."); + "and K is the class number."); + AddInput("Label", + "(Tensor, default: Tensor), The ground truth which is a 2-D " + "tensor. " + "If softLable is set to 0, Label is a Tensor with shape [N x " + "1]. " + "If softLable is set to 1, Label is a Tensor " + "with shape [N x K]."); AddOutput( "Softmax", "(Tensor, default: Tensor), A 2-D tensor with shape [N x K]. " @@ -163,15 +162,35 @@ class SoftmaxWithCrossEntropyOpGrad : public framework::OperatorWithKernel { } }; +class SoftmaxGradMaker : public framework::SingleGradOpDescMaker { + public: + using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; + + protected: + framework::OpDescBind Apply() const override { + framework::OpDescBind grad_op; + grad_op.SetType("softmax_with_cross_entropy_grad"); + grad_op.SetInput("Label", Input("Label")); + grad_op.SetInput("Softmax", Output("Softmax")); + grad_op.SetInput("Loss", Output("Loss")); + grad_op.SetInput(framework::GradVarName("Softmax"), OutputGrad("Softmax")); + grad_op.SetInput(framework::GradVarName("Loss"), OutputGrad("Loss")); + grad_op.SetOutput(framework::GradVarName("Logits"), InputGrad("Logits")); + grad_op.SetAttrMap(Attrs()); + return grad_op; + } +}; + } // namespace operators } // namespace paddle namespace ops = paddle::operators; -REGISTER_OP(softmax_with_cross_entropy, ops::SoftmaxWithCrossEntropyOp, - ops::SoftmaxWithCrossEntropyOpMaker, - softmax_with_cross_entropy_grad, - ops::SoftmaxWithCrossEntropyOpGrad); +REGISTER_OPERATOR(softmax_with_cross_entropy, ops::SoftmaxWithCrossEntropyOp, + ops::SoftmaxWithCrossEntropyOpMaker, + ops::SoftmaxWithCrossEntropyOpMaker); +REGISTER_OPERATOR(softmax_with_cross_entropy_grad, + ops::SoftmaxWithCrossEntropyOpGrad); REGISTER_OP_CPU_KERNEL(softmax_with_cross_entropy, ops::SoftmaxWithCrossEntropyKernel); REGISTER_OP_CPU_KERNEL(softmax_with_cross_entropy_grad, diff --git a/paddle/operators/sum_op.cc b/paddle/operators/sum_op.cc index 7c422b4770..5ae13492b3 100644 --- a/paddle/operators/sum_op.cc +++ b/paddle/operators/sum_op.cc @@ -45,10 +45,8 @@ class SumOpMaker : public framework::OpProtoAndCheckerMaker { public: SumOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("X", "the input tensors of sum operator.") - .AsDuplicable() - .NotInGradient(); - AddOutput("Out", "the output tensor of sum operator.").NotInGradient(); + AddInput("X", "the input tensors of sum operator.").AsDuplicable(); + AddOutput("Out", "the output tensor of sum operator."); AddComment(R"DOC( Sum the input tensors. @@ -58,23 +56,25 @@ or not. But the output only shares the LoD with the first input. } }; -class SumGradOp : public NetOp { +class SumGradMaker : public framework::GradOpDescMakerBase { public: - SumGradOp(const std::string& type, const framework::VariableNameMap& inputs, - const framework::VariableNameMap& outputs, - const framework::AttributeMap& attrs) - : NetOp(type, inputs, outputs, attrs) { - auto& x_grad_names = Outputs(framework::GradVarName("X")); - auto out_grad_name = this->Input(framework::GradVarName("Out")); + using framework::GradOpDescMakerBase::GradOpDescMakerBase; - framework::AttributeMap grad_attrs; - grad_attrs["scale"] = 1.0f; - for (auto& x_grad_name : x_grad_names) { - AppendOp(framework::OpRegistry::CreateOp( - "scale", {{"X", {out_grad_name}}}, {{"Out", {x_grad_name}}}, - grad_attrs)); - } - CompleteAddOp(false); + std::vector operator()() const override { + auto x_grads = InputGrad("X"); + std::vector grad_ops; + grad_ops.reserve(x_grads.size()); + auto og = OutputGrad("Out"); + std::transform(x_grads.begin(), x_grads.end(), std::back_inserter(grad_ops), + [&og](const std::string& x_grad) { + framework::OpDescBind grad_op; + grad_op.SetType("scale"); + grad_op.SetInput("X", og); + grad_op.SetOutput("Out", {x_grad}); + grad_op.SetAttr("scale", 1.0f); + return grad_op; + }); + return grad_ops; } }; @@ -82,5 +82,6 @@ class SumGradOp : public NetOp { } // namespace paddle namespace ops = paddle::operators; -REGISTER_OP(sum, ops::SumOp, ops::SumOpMaker, sum_grad, ops::SumGradOp); + +REGISTER_OPERATOR(sum, ops::SumOp, ops::SumOpMaker, ops::SumGradMaker); REGISTER_OP_CPU_KERNEL(sum, ops::SumKernel); From 2ccaec4f57afe94f36ee4781bae6e0eec78b29a8 Mon Sep 17 00:00:00 2001 From: zchen0211 Date: Mon, 2 Oct 2017 18:31:55 -0700 Subject: [PATCH 106/170] gather scatter cond --- paddle/operators/cond_op.cc | 5 ++--- paddle/operators/gather.h | 4 ++-- paddle/operators/gather_op.h | 4 ++-- paddle/operators/gather_test.cc | 4 +++- paddle/operators/scatter.h | 4 ++-- paddle/operators/scatter_op.h | 4 ++-- paddle/operators/scatter_test.cc | 4 +++- 7 files changed, 16 insertions(+), 13 deletions(-) diff --git a/paddle/operators/cond_op.cc b/paddle/operators/cond_op.cc index 55822827d9..7d7f1ba3b1 100644 --- a/paddle/operators/cond_op.cc +++ b/paddle/operators/cond_op.cc @@ -126,8 +126,7 @@ void CondOp::PrepareDataForSubnet( dim[0] = index_tensors[i].dims()[0]; tensor_child->mutable_data(dim, platform::CPUPlace()); - CPUGather(dev_ctx.GetPlace(), tensor_parent, &index_tensors[i], - tensor_child); + CPUGather(dev_ctx, tensor_parent, &index_tensors[i], tensor_child); } } @@ -188,7 +187,7 @@ void CondOp::MergeDataFromSubnet(const framework::Scope& scope, Variable* var_child = sub_scopes[i]->FindVar(output); PADDLE_ENFORCE_NOT_NULL(var_child); auto* tensor_child = &var_child->Get(); - ScatterAssign(dev_ctx.GetPlace(), tensor_child, &index_tensors[i], + ScatterAssign(dev_ctx, tensor_child, &index_tensors[i], tensor_parent); } } diff --git a/paddle/operators/gather.h b/paddle/operators/gather.h index cb635f6825..1e39a6da27 100644 --- a/paddle/operators/gather.h +++ b/paddle/operators/gather.h @@ -32,11 +32,11 @@ namespace operators { * return: output tensor */ template -void CPUGather(const platform::Place& place, +void CPUGather(const platform::DeviceContext& ctx, const paddle::framework::Tensor* src, const paddle::framework::Tensor* index, paddle::framework::Tensor* output) { - PADDLE_ENFORCE(platform::is_cpu_place(place)); + PADDLE_ENFORCE(platform::is_cpu_place(ctx.GetPlace())); // check index of shape 1-D PADDLE_ENFORCE(index->dims().size() == 1); int index_size = index->dims()[0]; diff --git a/paddle/operators/gather_op.h b/paddle/operators/gather_op.h index fb065b8da7..5bd2c36f7b 100644 --- a/paddle/operators/gather_op.h +++ b/paddle/operators/gather_op.h @@ -36,7 +36,7 @@ class GatherOpKernel : public framework::OpKernel { output->mutable_data(ctx.GetPlace()); - CPUGather(ctx.GetPlace(), x, index, output); + CPUGather(ctx.device_context(), x, index, output); } }; @@ -56,7 +56,7 @@ class GatherGradientOpKernel : public framework::OpKernel { auto place = ctx.GetEigenDevice(); dxt.device(place) = dxt.constant(static_cast(0)); - ScatterAssign(ctx.GetPlace(), dO, Index, dX); + ScatterAssign(ctx.device_context(), dO, Index, dX); } }; diff --git a/paddle/operators/gather_test.cc b/paddle/operators/gather_test.cc index 3c1d06ccd1..d8bf8dd9a4 100644 --- a/paddle/operators/gather_test.cc +++ b/paddle/operators/gather_test.cc @@ -41,7 +41,9 @@ TEST(Gather, GatherData) { int* p_output = output->mutable_data(make_ddim({2, 4}), CPUPlace()); - CPUGather(CPUPlace(), src, index, output); + auto* cpu_place = new paddle::platform::CPUPlace(); + paddle::platform::CPUDeviceContext ctx(*cpu_place); + CPUGather(ctx, src, index, output); for (int i = 0; i < 4; ++i) EXPECT_EQ(p_output[i], i + 4); for (int i = 4; i < 8; ++i) EXPECT_EQ(p_output[i], i - 4); diff --git a/paddle/operators/scatter.h b/paddle/operators/scatter.h index f895f22e28..0d174d3b5b 100644 --- a/paddle/operators/scatter.h +++ b/paddle/operators/scatter.h @@ -33,11 +33,11 @@ using Tensor = framework::Tensor; * return: output tensor */ template -void ScatterAssign(const platform::Place& place, +void ScatterAssign(const platform::DeviceContext& ctx, const paddle::framework::Tensor* src, const paddle::framework::Tensor* index, paddle::framework::Tensor* output) { - PADDLE_ENFORCE(platform::is_cpu_place(place)); + PADDLE_ENFORCE(platform::is_cpu_place(ctx.GetPlace())); // check index of shape 1-D PADDLE_ENFORCE(index->dims().size() == 1); int index_size = index->dims()[0]; diff --git a/paddle/operators/scatter_op.h b/paddle/operators/scatter_op.h index 771a1f2ddb..ac04968549 100644 --- a/paddle/operators/scatter_op.h +++ b/paddle/operators/scatter_op.h @@ -37,7 +37,7 @@ class ScatterOpKernel : public framework::OpKernel { // In place output: Out = Ref, Out[Index] += Updates Out->ShareDataWith(*Ref); // Apply ScatterUpdate: Out[index] += Updates[:] - ScatterAssign(ctx.GetPlace(), Updates, Index, Out); + ScatterAssign(ctx.device_context(), Updates, Index, Out); } }; @@ -56,7 +56,7 @@ class ScatterGradientOpKernel : public framework::OpKernel { dRef->ShareDataWith(*dOut); dUpdates->mutable_data(ctx.GetPlace()); // Gradient by Gather: dUpdates += dO[Index] - CPUGather(ctx.GetPlace(), dOut, Index, dUpdates); + CPUGather(ctx.device_context(), dOut, Index, dUpdates); } }; diff --git a/paddle/operators/scatter_test.cc b/paddle/operators/scatter_test.cc index bace6419d0..321bba3dad 100644 --- a/paddle/operators/scatter_test.cc +++ b/paddle/operators/scatter_test.cc @@ -40,7 +40,9 @@ TEST(scatter, ScatterUpdate) { float* p_output = output->mutable_data(make_ddim({4, 4}), CPUPlace()); - ScatterAssign(CPUPlace(), src, index, output); + auto* cpu_place = new paddle::platform::CPUPlace(); + paddle::platform::CPUDeviceContext ctx(*cpu_place); + ScatterAssign(ctx, src, index, output); for (size_t i = 0; i < 4; ++i) EXPECT_EQ(p_output[i], float(0)); for (size_t i = 0; i < 4; ++i) EXPECT_EQ(output->data()[i], float(0)); From b3e479da1c9cdb580e4577ebdafc5ec451ca4ed2 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Mon, 2 Oct 2017 18:38:49 -0700 Subject: [PATCH 107/170] Fix CI --- paddle/framework/grad_op_builder_test.cc | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/paddle/framework/grad_op_builder_test.cc b/paddle/framework/grad_op_builder_test.cc index 55c5fa420e..2dbc2e6620 100644 --- a/paddle/framework/grad_op_builder_test.cc +++ b/paddle/framework/grad_op_builder_test.cc @@ -39,28 +39,6 @@ class IOIgnoredOpMaker : public OpProtoAndCheckerMaker { namespace f = paddle::framework; -TEST(GradOpBuilder, AddTwo) { - std::shared_ptr add_op(f::OpRegistry::CreateOp( - "sum", {{"X", {"x", "y"}}}, {{"Out", {"out"}}}, {})); - std::shared_ptr grad_add_op = - f::OpRegistry::CreateGradOp(*add_op); - - EXPECT_EQ(grad_add_op->Inputs().size(), 1UL); - EXPECT_EQ(grad_add_op->Outputs().size(), 1UL); - EXPECT_EQ(grad_add_op->Input(f::GradVarName("Out")), f::GradVarName("out")); - auto &outputs = grad_add_op->Outputs(f::GradVarName("X")); - EXPECT_EQ(2UL, outputs.size()); - auto in_output = [&outputs](const std::string &name) { - for (auto &output_name : outputs) { - if (output_name == name) return true; - } - return false; - }; - - EXPECT_TRUE(in_output(f::GradVarName("x"))); - EXPECT_TRUE(in_output(f::GradVarName("y"))); -} - REGISTER_OP(mult_io, f::NOP, f::MutiInOutOpMaker, mult_io_grad, f::NOP); REGISTER_OP(io_ignored, f::NOP, f::IOIgnoredOpMaker, io_ignored_grad, f::NOP); From d1de7ec6304af675a666539c7991bf4bf242f738 Mon Sep 17 00:00:00 2001 From: Kexin Zhao Date: Mon, 2 Oct 2017 18:55:46 -0700 Subject: [PATCH 108/170] Change learning rate from attribute to input tensor --- paddle/operators/adagrad_op.cc | 7 ++++++- paddle/operators/adagrad_op.h | 9 ++++++++- python/paddle/v2/framework/tests/test_adagrad_op.py | 5 ++--- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/paddle/operators/adagrad_op.cc b/paddle/operators/adagrad_op.cc index 03e22cc600..56a5fbcb86 100644 --- a/paddle/operators/adagrad_op.cc +++ b/paddle/operators/adagrad_op.cc @@ -29,12 +29,17 @@ class AdagradOp : public framework::OperatorWithKernel { "Input(grad) of AdagradOp should not be null."); PADDLE_ENFORCE(ctx->HasInput("moment"), "Input(moment) of AdagradOp should not be null."); + PADDLE_ENFORCE(ctx->HasInput("learning_rate"), + "Input(learning_rate) of AdagradOp should not be null."); PADDLE_ENFORCE(ctx->HasOutput("param_out"), "Output(param_out) of AdagradOp should not be null."); PADDLE_ENFORCE(ctx->HasOutput("moment_out"), "Output(moment_out) of AdagradOp should not be null."); + auto lr_dims = ctx->GetInputDim("learning_rate"); + PADDLE_ENFORCE_EQ(framework::product(lr_dims), 1, + "learning_rate should have one element"); auto param_dim = ctx->GetInputDim("param"); PADDLE_ENFORCE_EQ( param_dim, ctx->GetInputDim("grad"), @@ -56,11 +61,11 @@ class AdagradOpMaker : public framework::OpProtoAndCheckerMaker { AddInput("param", "Input parameter"); AddInput("grad", "Input gradient"); AddInput("moment", "Second moment"); + AddInput("learning_rate", "learning rate of adagrad"); AddOutput("param_out", "Output parameter"); AddOutput("moment_out", "Output second moment"); - AddAttr("learning_rate", "Learning rate"); AddAttr("epsilon", "Constant for numerical stability"); AddComment(R"DOC( diff --git a/paddle/operators/adagrad_op.h b/paddle/operators/adagrad_op.h index ca1836c3fa..73833d4a3f 100644 --- a/paddle/operators/adagrad_op.h +++ b/paddle/operators/adagrad_op.h @@ -20,6 +20,11 @@ namespace paddle { namespace operators { using Tensor = framework::Tensor; + +template +using EigenScalar = framework::EigenScalar; + template using EigenVector = framework::EigenVector; @@ -34,12 +39,14 @@ class AdagradOpKernel : public framework::OpKernel { param_out->mutable_data(ctx.GetPlace()); moment_out->mutable_data(ctx.GetPlace()); - float lr = ctx.Attr("learning_rate"); + float lr = ctx.Input("learning_rate")->data()[0]; float epsilon = ctx.Attr("epsilon"); auto p = EigenVector::Flatten(*ctx.Input("param")); auto g = EigenVector::Flatten(*ctx.Input("grad")); auto m = EigenVector::Flatten(*ctx.Input("moment")); + auto lr = EigenScalar::From(*ctx.Input("learning_rate")); + auto p_out = EigenVector::Flatten(*param_out); auto m_out = EigenVector::Flatten(*moment_out); auto place = ctx.GetEigenDevice(); diff --git a/python/paddle/v2/framework/tests/test_adagrad_op.py b/python/paddle/v2/framework/tests/test_adagrad_op.py index b3f8b812e1..2ee38ea37c 100644 --- a/python/paddle/v2/framework/tests/test_adagrad_op.py +++ b/python/paddle/v2/framework/tests/test_adagrad_op.py @@ -11,7 +11,7 @@ class TestAdagradOp(OpTest): grad = np.random.random((123, 321)).astype("float32") moment = np.zeros((123, 321)).astype("float32") - learning_rate = 0.01 + lr = np.array([0.01]).astype("float32") epsilon = 1e-6 self.inputs = {'param': param, 'grad': grad, 'moment': moment} @@ -19,8 +19,7 @@ class TestAdagradOp(OpTest): self.attrs = {'learning_rate': learning_rate, 'epsilon': epsilon} moment_out = moment + grad * grad - param_out = param - learning_rate * grad / (np.sqrt(moment_out) + - epsilon) + param_out = param - lr * grad / (np.sqrt(moment_out) + epsilon) self.outputs = {'param_out': param_out, 'moment_out': moment_out} From 163d28714349d51596be1cb165f93be2b8290bda Mon Sep 17 00:00:00 2001 From: Kavya Srinet Date: Mon, 2 Oct 2017 19:23:05 -0700 Subject: [PATCH 109/170] Made learning rate the input --- paddle/operators/rmsprop_op.cc | 16 +++++++++++----- paddle/operators/rmsprop_op.h | 2 +- .../paddle/v2/framework/tests/test_rmsprop_op.py | 15 ++++++++------- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/paddle/operators/rmsprop_op.cc b/paddle/operators/rmsprop_op.cc index dcf3599f4d..602efab3db 100644 --- a/paddle/operators/rmsprop_op.cc +++ b/paddle/operators/rmsprop_op.cc @@ -24,11 +24,13 @@ class RmspropOp : public framework::OperatorWithKernel { protected: void InferShape(framework::InferShapeContextBase *ctx) const override { PADDLE_ENFORCE(ctx->HasInput("Param"), - "Input(param) of RmspropOp should not be null."); + "Input(Param) of RmspropOp should not be null."); PADDLE_ENFORCE(ctx->HasInput("Grad"), - "Input(grad) of RmspropOp should not be null."); + "Input(Grad) of RmspropOp should not be null."); PADDLE_ENFORCE(ctx->HasInput("Moment"), - "Input(moment) of RmspropOp should not be null."); + "Input(Moment) of RmspropOp should not be null."); + PADDLE_ENFORCE(ctx->HasInput("LearningRate"), + "Input(LearningRate) of RmspropOp should not be null."); PADDLE_ENFORCE(ctx->HasOutput("ParamOut"), "Output(param_out) of RmspropOp should not be null."); @@ -43,6 +45,10 @@ class RmspropOp : public framework::OperatorWithKernel { param_dim, ctx->GetInputDim("Moment"), "Param and moment input of RmspropOp should have the same dimension."); + auto lr_dim = ctx->GetInputDim("LearningRate"); + PADDLE_ENFORCE_EQ(framework::product(lr_dim), 1, + "Learning Rate should be a scalar."); + ctx->SetOutputDim("ParamOut", param_dim); ctx->SetOutputDim("MomentOut", param_dim); } @@ -56,11 +62,11 @@ class RmspropOpMaker : public framework::OpProtoAndCheckerMaker { AddInput("Param", "Input parameter"); AddInput("Grad", "Input gradient"); AddInput("Moment", "Second moment"); + AddInput("LearningRate", "Learning Rate"); AddOutput("ParamOut", "Output parameter"); AddOutput("MomentOut", "Output second moment"); - AddAttr("learningRate", "Learning rate"); AddAttr("epsilon", "Constant for numerical stability"); AddAttr("decayRate", "Decay rate for moving average of gradients"); AddComment(R"DOC( @@ -68,7 +74,7 @@ class RmspropOpMaker : public framework::OpProtoAndCheckerMaker { RMSprop MomentOut = decayRate * Moment + (1 - decayRate) * Grad * Grad -ParamOut = Param - learningRate * Grad / (sqrt(MomentOut) + epsilon) +ParamOut = Param - LearningRate * Grad / (sqrt(MomentOut) + epsilon) The original slide(Slide 29 of http://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf) diff --git a/paddle/operators/rmsprop_op.h b/paddle/operators/rmsprop_op.h index c94c24bddd..65b9edd35b 100644 --- a/paddle/operators/rmsprop_op.h +++ b/paddle/operators/rmsprop_op.h @@ -34,13 +34,13 @@ class RmspropOpKernel : public framework::OpKernel { param_out->mutable_data(ctx.GetPlace()); moment_out->mutable_data(ctx.GetPlace()); - float lr = ctx.Attr("learningRate"); float epsilon = ctx.Attr("epsilon"); float decay = ctx.Attr("decayRate"); auto p = EigenVector::Flatten(*ctx.Input("Param")); auto g = EigenVector::Flatten(*ctx.Input("Grad")); auto m = EigenVector::Flatten(*ctx.Input("Moment")); + float lr = ctx.Input("LearningRate")->data()[0]; auto p_out = EigenVector::Flatten(*param_out); auto m_out = EigenVector::Flatten(*moment_out); auto place = ctx.GetEigenDevice(); diff --git a/python/paddle/v2/framework/tests/test_rmsprop_op.py b/python/paddle/v2/framework/tests/test_rmsprop_op.py index 1fc59a0f11..64ca5da48e 100644 --- a/python/paddle/v2/framework/tests/test_rmsprop_op.py +++ b/python/paddle/v2/framework/tests/test_rmsprop_op.py @@ -10,19 +10,20 @@ class TestRmspropOp(OpTest): param = np.random.random((123, 321)).astype("float32") grad = np.random.random((123, 321)).astype("float32") moment = np.zeros((123, 321)).astype("float32") + learning_rate = np.array([0.01]).astype("float32") - learning_rate = 0.01 epsilon = 1e-6 decay_rate = 0.9 - self.inputs = {'Param': param, 'Grad': grad, 'Moment': moment} - - self.attrs = { - 'learningRate': learning_rate, - 'epsilon': epsilon, - 'decayRate': decay_rate + self.inputs = { + 'Param': param, + 'Grad': grad, + 'Moment': moment, + 'LearningRate': learning_rate } + self.attrs = {'epsilon': epsilon, 'decayRate': decay_rate} + moment_out = decay_rate * moment + (1 - decay_rate) * grad * grad param_out = param - learning_rate * grad / (np.sqrt(moment_out) + epsilon) From 42e7fe05a23067677fe7cf552e9534e329886fbb Mon Sep 17 00:00:00 2001 From: Abhinav Arora Date: Mon, 2 Oct 2017 20:08:06 -0700 Subject: [PATCH 110/170] Changing learning rate from attribute to input(float) (#4568) * Changing learning rate from attribute to input(float) * Removing obsolete code --- paddle/operators/sgd_op.cc | 4 +++- paddle/operators/sgd_op.h | 2 +- paddle/pybind/pybind.cc | 7 +++++++ python/paddle/v2/framework/tests/op_test.py | 17 +++++++++++------ python/paddle/v2/framework/tests/test_sgd_op.py | 3 +-- 5 files changed, 23 insertions(+), 10 deletions(-) diff --git a/paddle/operators/sgd_op.cc b/paddle/operators/sgd_op.cc index 3bce95535c..8f9eae4186 100644 --- a/paddle/operators/sgd_op.cc +++ b/paddle/operators/sgd_op.cc @@ -27,6 +27,8 @@ class SGDOp : public framework::OperatorWithKernel { "Input(param) of SGDOp should not be null."); PADDLE_ENFORCE(ctx->HasInput("grad"), "Input(grad) of SGDOp should not be null."); + PADDLE_ENFORCE(ctx->HasInput("learning_rate"), + "Input(learning_rate) of SGDOp should not be null."); PADDLE_ENFORCE(ctx->HasOutput("param_out"), "Output(param_out) of SGDOp should not be null."); @@ -42,9 +44,9 @@ class SGDOpMaker : public framework::OpProtoAndCheckerMaker { SGDOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { AddInput("param", "input parameter"); + AddInput("learning_rate", "learning rate of sgd"); AddInput("grad", "input gradient"); AddOutput("param_out", "output parameter"); - AddAttr("learning_rate", "learning rate of sgd"); AddComment(R"DOC( Simplest sgd algorithm. diff --git a/paddle/operators/sgd_op.h b/paddle/operators/sgd_op.h index a3fe330894..977d201ced 100644 --- a/paddle/operators/sgd_op.h +++ b/paddle/operators/sgd_op.h @@ -31,7 +31,7 @@ class SGDOpKernel : public framework::OpKernel { auto param = ctx.Input("param"); auto grad = ctx.Input("grad"); auto param_out = ctx.Output("param_out"); - float lr = ctx.Attr("learning_rate"); + float lr = *ctx.Input("learning_rate"); param_out->mutable_data(ctx.GetPlace()); diff --git a/paddle/pybind/pybind.cc b/paddle/pybind/pybind.cc index f4121e9d71..d480427f59 100644 --- a/paddle/pybind/pybind.cc +++ b/paddle/pybind/pybind.cc @@ -143,6 +143,13 @@ All parameter, weight, gradient are variables in Paddle. .def("set_int", [](Variable &var, int val) -> void { *var.GetMutable() = val; }) .def("get_int", [](const Variable &var) -> int { return var.Get(); }) + .def("is_float", [](const Variable &var) { return var.IsType(); }) + .def("set_float", + [](Variable &var, float val) -> void { + *var.GetMutable() = val; + }) + .def("get_float", + [](const Variable &var) -> float { return var.Get(); }) .def("get_tensor", [](Variable &self) -> LoDTensor * { return self.GetMutable(); diff --git a/python/paddle/v2/framework/tests/op_test.py b/python/paddle/v2/framework/tests/op_test.py index 75df2eeddf..81067f38bb 100644 --- a/python/paddle/v2/framework/tests/op_test.py +++ b/python/paddle/v2/framework/tests/op_test.py @@ -46,12 +46,17 @@ def create_op(scope, op_type, inputs, outputs, attrs): def set_input(scope, op, inputs, place): def __set_input__(var_name, var): - tensor = scope.find_var(var_name).get_tensor() - if isinstance(var, tuple): - tensor.set_lod(var[1]) - var = var[0] - tensor.set_dims(var.shape) - tensor.set(var, place) + if isinstance(var, tuple) or isinstance(var, np.ndarray): + tensor = scope.find_var(var_name).get_tensor() + if isinstance(var, tuple): + tensor.set_lod(var[1]) + var = var[0] + tensor.set_dims(var.shape) + tensor.set(var, place) + elif isinstance(var, float): + scope.find_var(var_name).set_float(var) + elif isinstance(var, int): + scope.find_var(var_name).set_int(var) for in_name, in_dup in Operator.get_op_inputs(op.type()): if in_name in inputs: diff --git a/python/paddle/v2/framework/tests/test_sgd_op.py b/python/paddle/v2/framework/tests/test_sgd_op.py index 64e54d1500..f1125f4edb 100644 --- a/python/paddle/v2/framework/tests/test_sgd_op.py +++ b/python/paddle/v2/framework/tests/test_sgd_op.py @@ -10,8 +10,7 @@ class TestSGDOp(OpTest): g = np.random.random((102, 105)).astype("float32") lr = 0.1 - self.inputs = {'param': w, 'grad': g} - self.attrs = {'learning_rate': lr} + self.inputs = {'param': w, 'grad': g, 'learning_rate': lr} self.outputs = {'param_out': w - lr * g} def test_check_output(self): From ff7fdb7d705a34e224561cb53933b5477fef644b Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Tue, 3 Oct 2017 10:28:52 -0700 Subject: [PATCH 111/170] Add `CreateBackwardOp` function --- paddle/framework/backward.cc | 56 +++++++++++++++++++++++++++++++++ paddle/framework/backward.h | 5 +++ paddle/framework/op_desc.cc | 20 ++++++++++++ paddle/framework/op_desc.h | 7 +++++ paddle/framework/op_registry.cc | 5 +++ paddle/framework/op_registry.h | 2 ++ 6 files changed, 95 insertions(+) diff --git a/paddle/framework/backward.cc b/paddle/framework/backward.cc index 0ec18de5b8..1b4c5c025e 100644 --- a/paddle/framework/backward.cc +++ b/paddle/framework/backward.cc @@ -222,5 +222,61 @@ std::unique_ptr Backward( return BackwardRecursive(forwardOp, no_grad_names, uid); } +// ==================================== // + +static bool AllGradInSet(const std::vector& names, + const std::unordered_set& set) { + for (const std::string& name : names) { + if (!set.count(GradVarName(name))) { + return false; + } + } + return true; +} + +std::vector CreatBackwardOps( + const OpDescBind& op_desc, unordered_map& no_grad_vars) { + std::vector grad_op_descs; + // All input gradients of forwarding operator do not need to calculat. + if (AllGradInSet(op_desc_.InputNames(), kGradVarSuffix, no_grad_vars)) { + return grad_op_descs; // empty vector + } + // All output gradients of forwarding operator do not need to calculate. + const std::vector& outputs = op_desc_.OutputNames(); + if (AllGradInSet(outputs, kGradVarSuffix, no_grad_vars)) { + for (const std::string& name : outputs) { + no_grad_vars.insert(GradVarName(name)); + } + return grad_op_descs; // empty vector + } + + grad_op_descs = OpRegistry::CreateGradOpDescs(op_desc); + + std::vector fill_zeros_ops; + for (OpDescBind& desc : grad_op_descs) { + for (const std::string& in_name : desc.InputNames()) { + if (no_grad_vars.count(in_name)) { + std::string prefix = in_name.substr( + 0, in_name.size() - sizeof(kGradVarSuffix) / sizeof(char) + 1); + std::string new_name = prefix + kZeroVarSuffix; + desc.Rename(in_name, new_name); + OpDescBind op_desc_bind( + {"fill_zeros_like", {{"X", {prefix}}}, {{"Y", {new_name}}}, {}}); + fill_zeros_ops.push_back(op_desc_bind); + } + } + for (const std::string& out_name : desc.OutputName()) { + if (no_grad_vars.count(out_name)) { + desc.Rename(out_name, kEmptyVarName); + } + } + } + grad_op_descs.insert(grad_op_descs.begin(), fill_zeros_ops.begin(), + fill_zeros_ops.end()); + + // TODO (fengjiayi): RNN op + return grad_op_descs; +} + } // namespace framework } // namespace paddle diff --git a/paddle/framework/backward.h b/paddle/framework/backward.h index 1ecf69881b..6aeddafb41 100644 --- a/paddle/framework/backward.h +++ b/paddle/framework/backward.h @@ -23,5 +23,10 @@ namespace framework { extern std::unique_ptr Backward( const OperatorBase& forwardOp, const std::unordered_set& no_grad_vars); + +extern void AppendBackwardOps( + BlockDescBind& block_desc, + const std::unordered_set& no_grad_vars); + } // namespace framework } // namespace paddle diff --git a/paddle/framework/op_desc.cc b/paddle/framework/op_desc.cc index 0c12c55dc0..e98f8f1154 100644 --- a/paddle/framework/op_desc.cc +++ b/paddle/framework/op_desc.cc @@ -18,6 +18,15 @@ limitations under the License. */ namespace paddle { namespace framework { +OpDescBind::OpDescBind(const std::string &type, const VariableNameMap &inputs, + const VariableNameMap &outputs, + const AttributeMap &attrs) { + op_desc_.set_type(type); + inputs_ = inputs; + outputs_ = outputs; + attrs_ = attrs; +} + OpDesc *OpDescBind::Proto() { Sync(); return &op_desc_; @@ -112,6 +121,17 @@ const std::unordered_map &OpDescBind::GetAttrMap() return attrs_; } +void Rename(const std::string &old_name, const std::string &new_name) { + for (std : string &input : inputs_) { + std::replace(input.second.begin(), input.second.end(), old_name, new_name); + } + for (std::string &output : outputs_) { + std::repalce(output.second.begin(), output.second.end(), old_name, + new_name); + } + need_update_ = true; +} + void OpDescBind::Sync() { if (need_update_) { this->op_desc_.mutable_inputs()->Clear(); diff --git a/paddle/framework/op_desc.h b/paddle/framework/op_desc.h index 0cf7d13971..a32e6d03f7 100644 --- a/paddle/framework/op_desc.h +++ b/paddle/framework/op_desc.h @@ -26,6 +26,11 @@ class BlockDescBind; class OpDescBind { public: + OpDescBind() {} + + OpDescBind(const std::string &type, const VariableNameMap &inputs, + const VariableNameMap &outputs, const AttributeMap &attrs); + OpDesc *Proto(); std::string Type() const { return op_desc_.type(); } @@ -67,6 +72,8 @@ class OpDescBind { int GetBlockAttr(const std::string &name) const; + void Rename(const std::string &old_name, const std::string &new_name); + // Only be used in C++ const std::unordered_map &GetAttrMap() const; diff --git a/paddle/framework/op_registry.cc b/paddle/framework/op_registry.cc index b0e85dd49f..fe3228ce5b 100644 --- a/paddle/framework/op_registry.cc +++ b/paddle/framework/op_registry.cc @@ -57,5 +57,10 @@ std::unique_ptr OpRegistry::CreateGradOp(const OperatorBase& op) { return std::unique_ptr(BuildGradOp(&op)); } +static std::vector CreateGradOpDescs(const OpDescBind& op_desc) { + auto& info = OpInfoMap::Instance().Get(op_desc.Type()); + return info.grad_op_maker_(op_desc); +} + } // namespace framework } // namespace paddle diff --git a/paddle/framework/op_registry.h b/paddle/framework/op_registry.h index 4ee2c7d275..c80b6e9630 100644 --- a/paddle/framework/op_registry.h +++ b/paddle/framework/op_registry.h @@ -68,6 +68,8 @@ class OpRegistry { static std::unique_ptr CreateOp(const OpDesc& op_desc); static std::unique_ptr CreateGradOp(const OperatorBase& op); + + static std::vector CreateGradOpDescs(const OpDescBind& op_desc); }; class Registrar { From 494b3bda7d784315433b85826c9cbd18cac5723a Mon Sep 17 00:00:00 2001 From: dongzhihong Date: Tue, 3 Oct 2017 10:28:57 -0700 Subject: [PATCH 112/170] fix backward test case --- paddle/framework/backward.cc | 2 +- paddle/framework/backward_test.cc | 15 +++++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/paddle/framework/backward.cc b/paddle/framework/backward.cc index 35759f8e78..2c13ddd8d0 100644 --- a/paddle/framework/backward.cc +++ b/paddle/framework/backward.cc @@ -159,7 +159,7 @@ static std::unique_ptr BackwardRecursive( insert_position.push_back( {dup_op.back(), OpRegistry::CreateOp( - "add", {{"X", {insert_add_x}}, {"X", {insert_add_y}}}, + "sum", {{"X", {insert_add_x}}, {"X", {insert_add_y}}}, {{"Out", {insert_add_out}}}, {})}); } } diff --git a/paddle/framework/backward_test.cc b/paddle/framework/backward_test.cc index 6932f5b989..a36e7bde8c 100644 --- a/paddle/framework/backward_test.cc +++ b/paddle/framework/backward_test.cc @@ -133,15 +133,18 @@ class FillZeroOpMaker : public OpProtoAndCheckerMaker { } }; -class AddOpMaker : public OpProtoAndCheckerMaker { +class SumOpMaker : public framework::OpProtoAndCheckerMaker { public: - AddOpMaker(OpProto *proto, OpAttrChecker *op_checker) + SumOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("X", "x").AsDuplicable(); - AddOutput("Out", "out"); + AddInput("X", "the input tensors of sum operator.") + .AsDuplicable() + .NotInGradient(); + AddOutput("Out", "the output tensor of sum operator.").NotInGradient(); AddComment(""); } }; + } // namespace framework } // namespace paddle @@ -154,7 +157,7 @@ REGISTER_OP(mul, f::NOP, f::MulOpMaker, mul_grad, f::NOP); REGISTER_OP(sigmoid, f::NOP, f::SigmoidOpMaker, sigmoid_grad, f::NOP); REGISTER_OP_WITHOUT_GRADIENT(nograd, f::NOP, f::NoGradOpMaker); REGISTER_OP_WITHOUT_GRADIENT(fill_zeros_like, f::NOP, f::FillZeroOpMaker); -REGISTER_OP(add, f::NOP, f::AddOpMaker, add_grad, f::NOP); +REGISTER_OP(sum, f::NOP, f::SumOpMaker, sum_grad, f::NOP); REGISTER_OP_WITHOUT_GRADIENT(fc, f::FcOp, f::FcOpMaker); REGISTER_OP(many_output_op, f::NOP, f::ManyOutputOpMaker, many_output_op_grad, f::NOP); @@ -283,7 +286,7 @@ TEST(Backward, net_shared_weight) { ASSERT_TRUE(bwd->IsNetOp()); auto bwd_net = static_cast(bwd.get()); ASSERT_EQ(3UL, bwd_net->ops_.size()); - ASSERT_EQ("add", bwd_net->ops_[2]->Type()); + ASSERT_EQ("sum", bwd_net->ops_[2]->Type()); } TEST(Backward, op_register_grad_not_for_network) { From 2d876b864395513de8db52db944ee5e8150d2730 Mon Sep 17 00:00:00 2001 From: zchen0211 Date: Tue, 3 Oct 2017 10:54:22 -0700 Subject: [PATCH 113/170] gather scatter fix according to google style --- paddle/operators/cond_op.cc | 4 ++-- paddle/operators/gather.cu.h | 14 +++++++------- paddle/operators/gather.h | 18 +++++++++--------- paddle/operators/gather_op.cu | 4 ++-- paddle/operators/gather_op.h | 4 ++-- paddle/operators/gather_test.cc | 2 +- paddle/operators/scatter.cu.h | 18 +++++++++--------- paddle/operators/scatter.h | 16 +++++++--------- paddle/operators/scatter_op.cu | 4 ++-- paddle/operators/scatter_op.h | 4 ++-- paddle/operators/scatter_test.cc | 2 +- 11 files changed, 44 insertions(+), 46 deletions(-) diff --git a/paddle/operators/cond_op.cc b/paddle/operators/cond_op.cc index 7d7f1ba3b1..2737104a20 100644 --- a/paddle/operators/cond_op.cc +++ b/paddle/operators/cond_op.cc @@ -126,7 +126,7 @@ void CondOp::PrepareDataForSubnet( dim[0] = index_tensors[i].dims()[0]; tensor_child->mutable_data(dim, platform::CPUPlace()); - CPUGather(dev_ctx, tensor_parent, &index_tensors[i], tensor_child); + CPUGather(dev_ctx, *tensor_parent, index_tensors[i], tensor_child); } } @@ -187,7 +187,7 @@ void CondOp::MergeDataFromSubnet(const framework::Scope& scope, Variable* var_child = sub_scopes[i]->FindVar(output); PADDLE_ENFORCE_NOT_NULL(var_child); auto* tensor_child = &var_child->Get(); - ScatterAssign(dev_ctx, tensor_child, &index_tensors[i], + ScatterAssign(dev_ctx, *tensor_child, index_tensors[i], tensor_parent); } } diff --git a/paddle/operators/gather.cu.h b/paddle/operators/gather.cu.h index 2ae11376a2..8d04ecd284 100644 --- a/paddle/operators/gather.cu.h +++ b/paddle/operators/gather.cu.h @@ -46,14 +46,14 @@ __global__ void GatherCUDAKernel(const T* params, const int* indices, T* output, * return: output tensor */ template -void GPUGather(const platform::DeviceContext& ctx, const Tensor* src, - const Tensor* index, Tensor* output) { +void GPUGather(const platform::DeviceContext& ctx, const Tensor& src, + const Tensor& index, Tensor* output) { // PADDLE_ENFORCE(platform::is_gpu_place(place)); // check index of shape 1-D - PADDLE_ENFORCE(index->dims().size() == 1); - int index_size = index->dims()[0]; + PADDLE_ENFORCE(index.dims().size() == 1); + int index_size = index.dims()[0]; - auto src_dims = src->dims(); + auto src_dims = src.dims(); framework::DDim output_dims(src_dims); output_dims[0] = index_size; @@ -61,8 +61,8 @@ void GPUGather(const platform::DeviceContext& ctx, const Tensor* src, int slice_size = 1; for (int i = 1; i < src_dims.size(); ++i) slice_size *= src_dims[i]; - const T* p_src = src->data(); - const int* p_index = index->data(); + const T* p_src = src.data(); + const int* p_index = index.data(); T* p_output = output->data(); int block = 512; diff --git a/paddle/operators/gather.h b/paddle/operators/gather.h index 1e39a6da27..052db49cb3 100644 --- a/paddle/operators/gather.h +++ b/paddle/operators/gather.h @@ -24,6 +24,8 @@ limitations under the License. */ namespace paddle { namespace operators { +using framework::Tensor; + /** * A thin wrapper for gathering on cpu tensor * Return a new tensor from source tensor, gathered according to index @@ -32,21 +34,19 @@ namespace operators { * return: output tensor */ template -void CPUGather(const platform::DeviceContext& ctx, - const paddle::framework::Tensor* src, - const paddle::framework::Tensor* index, - paddle::framework::Tensor* output) { +void CPUGather(const platform::DeviceContext& ctx, const Tensor& src, + const Tensor& index, Tensor* output) { PADDLE_ENFORCE(platform::is_cpu_place(ctx.GetPlace())); // check index of shape 1-D - PADDLE_ENFORCE(index->dims().size() == 1); - int index_size = index->dims()[0]; + PADDLE_ENFORCE(index.dims().size() == 1); + int index_size = index.dims()[0]; - auto src_dims = src->dims(); + auto src_dims = src.dims(); framework::DDim output_dims(src_dims); output_dims[0] = index_size; - const T* p_src = src->data(); - const int* p_index = index->data(); + const T* p_src = src.data(); + const int* p_index = index.data(); T* p_output = output->data(); // slice size diff --git a/paddle/operators/gather_op.cu b/paddle/operators/gather_op.cu index 9937be5915..92219d6a43 100644 --- a/paddle/operators/gather_op.cu +++ b/paddle/operators/gather_op.cu @@ -32,7 +32,7 @@ class GatherOpCUDAKernel : public framework::OpKernel { output->mutable_data(ctx.GetPlace()); - GPUGather(ctx.device_context(), x, index, output); + GPUGather(ctx.device_context(), *x, *index, output); } }; @@ -52,7 +52,7 @@ class GatherGradOpCUDAKernel : public framework::OpKernel { auto place = ctx.GetEigenDevice(); dxt.device(place) = dxt.constant(static_cast(0)); - GPUScatterAssign(ctx.device_context(), dO, Index, dX); + GPUScatterAssign(ctx.device_context(), *dO, *Index, dX); } }; diff --git a/paddle/operators/gather_op.h b/paddle/operators/gather_op.h index 5bd2c36f7b..8276ed0d3d 100644 --- a/paddle/operators/gather_op.h +++ b/paddle/operators/gather_op.h @@ -36,7 +36,7 @@ class GatherOpKernel : public framework::OpKernel { output->mutable_data(ctx.GetPlace()); - CPUGather(ctx.device_context(), x, index, output); + CPUGather(ctx.device_context(), *x, *index, output); } }; @@ -56,7 +56,7 @@ class GatherGradientOpKernel : public framework::OpKernel { auto place = ctx.GetEigenDevice(); dxt.device(place) = dxt.constant(static_cast(0)); - ScatterAssign(ctx.device_context(), dO, Index, dX); + ScatterAssign(ctx.device_context(), *dO, *Index, dX); } }; diff --git a/paddle/operators/gather_test.cc b/paddle/operators/gather_test.cc index d8bf8dd9a4..cbd86b8796 100644 --- a/paddle/operators/gather_test.cc +++ b/paddle/operators/gather_test.cc @@ -43,7 +43,7 @@ TEST(Gather, GatherData) { auto* cpu_place = new paddle::platform::CPUPlace(); paddle::platform::CPUDeviceContext ctx(*cpu_place); - CPUGather(ctx, src, index, output); + CPUGather(ctx, *src, *index, output); for (int i = 0; i < 4; ++i) EXPECT_EQ(p_output[i], i + 4); for (int i = 4; i < 8; ++i) EXPECT_EQ(p_output[i], i - 4); diff --git a/paddle/operators/scatter.cu.h b/paddle/operators/scatter.cu.h index f4a3965d94..d95436be4f 100644 --- a/paddle/operators/scatter.cu.h +++ b/paddle/operators/scatter.cu.h @@ -19,6 +19,8 @@ namespace paddle { namespace operators { +using Tensor = framework::Tensor; + #define CUDA_1D_KERNEL_LOOP(i, n) \ for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < (n); \ i += blockDim.x * gridDim.x) @@ -45,16 +47,14 @@ __global__ void ScatterCUDAKernel(const T* params, const int* indices, * return: output tensor */ template -void GPUScatterAssign(const platform::DeviceContext& ctx, - const paddle::framework::Tensor* src, - const paddle::framework::Tensor* index, - paddle::framework::Tensor* output) { +void GPUScatterAssign(const platform::DeviceContext& ctx, const Tensor& src, + const Tensor& index, Tensor* output) { // PADDLE_ENFORCE(platform::is_gpu_place(place)); // check index of shape 1-D - PADDLE_ENFORCE(index->dims().size() == 1); - int index_size = index->dims()[0]; + PADDLE_ENFORCE(index.dims().size() == 1); + int index_size = index.dims()[0]; - auto src_dims = src->dims(); + auto src_dims = src.dims(); framework::DDim output_dims(src_dims); output_dims[0] = index_size; @@ -62,8 +62,8 @@ void GPUScatterAssign(const platform::DeviceContext& ctx, int slice_size = 1; for (int i = 1; i < src_dims.size(); ++i) slice_size *= src_dims[i]; - const T* p_src = src->data(); - const int* p_index = index->data(); + const T* p_src = src.data(); + const int* p_index = index.data(); T* p_output = output->data(); int block = 512; diff --git a/paddle/operators/scatter.h b/paddle/operators/scatter.h index 0d174d3b5b..c1fb844ebd 100644 --- a/paddle/operators/scatter.h +++ b/paddle/operators/scatter.h @@ -33,20 +33,18 @@ using Tensor = framework::Tensor; * return: output tensor */ template -void ScatterAssign(const platform::DeviceContext& ctx, - const paddle::framework::Tensor* src, - const paddle::framework::Tensor* index, - paddle::framework::Tensor* output) { +void ScatterAssign(const platform::DeviceContext& ctx, const Tensor& src, + const Tensor& index, Tensor* output) { PADDLE_ENFORCE(platform::is_cpu_place(ctx.GetPlace())); // check index of shape 1-D - PADDLE_ENFORCE(index->dims().size() == 1); - int index_size = index->dims()[0]; + PADDLE_ENFORCE(index.dims().size() == 1); + int index_size = index.dims()[0]; - auto src_dims = src->dims(); + auto src_dims = src.dims(); auto dst_dims = output->dims(); - const T* p_src = src->data(); - const int* p_index = index->data(); + const T* p_src = src.data(); + const int* p_index = index.data(); T* p_output = output->data(); // check src shape and dst shape should match diff --git a/paddle/operators/scatter_op.cu b/paddle/operators/scatter_op.cu index 6d13a876f9..06f4d75944 100644 --- a/paddle/operators/scatter_op.cu +++ b/paddle/operators/scatter_op.cu @@ -32,7 +32,7 @@ class ScatterOpCUDAKernel : public framework::OpKernel { Out->ShareDataWith(*Ref); - GPUScatterAssign(ctx.device_context(), Updates, Index, Out); + GPUScatterAssign(ctx.device_context(), *Updates, *Index, Out); } }; @@ -51,7 +51,7 @@ class ScatterGradOpCUDAKernel : public framework::OpKernel { dRef->ShareDataWith(*dOut); dUpdates->mutable_data(ctx.GetPlace()); // Gradient by Gather: dUpdates = dO[Index] - GPUGather(ctx.device_context(), dOut, Index, dUpdates); + GPUGather(ctx.device_context(), *dOut, *Index, dUpdates); } }; diff --git a/paddle/operators/scatter_op.h b/paddle/operators/scatter_op.h index ac04968549..6101219006 100644 --- a/paddle/operators/scatter_op.h +++ b/paddle/operators/scatter_op.h @@ -37,7 +37,7 @@ class ScatterOpKernel : public framework::OpKernel { // In place output: Out = Ref, Out[Index] += Updates Out->ShareDataWith(*Ref); // Apply ScatterUpdate: Out[index] += Updates[:] - ScatterAssign(ctx.device_context(), Updates, Index, Out); + ScatterAssign(ctx.device_context(), *Updates, *Index, Out); } }; @@ -56,7 +56,7 @@ class ScatterGradientOpKernel : public framework::OpKernel { dRef->ShareDataWith(*dOut); dUpdates->mutable_data(ctx.GetPlace()); // Gradient by Gather: dUpdates += dO[Index] - CPUGather(ctx.device_context(), dOut, Index, dUpdates); + CPUGather(ctx.device_context(), *dOut, *Index, dUpdates); } }; diff --git a/paddle/operators/scatter_test.cc b/paddle/operators/scatter_test.cc index 321bba3dad..00dbdacbfe 100644 --- a/paddle/operators/scatter_test.cc +++ b/paddle/operators/scatter_test.cc @@ -42,7 +42,7 @@ TEST(scatter, ScatterUpdate) { auto* cpu_place = new paddle::platform::CPUPlace(); paddle::platform::CPUDeviceContext ctx(*cpu_place); - ScatterAssign(ctx, src, index, output); + ScatterAssign(ctx, *src, *index, output); for (size_t i = 0; i < 4; ++i) EXPECT_EQ(p_output[i], float(0)); for (size_t i = 0; i < 4; ++i) EXPECT_EQ(output->data()[i], float(0)); From 3395bf7ad0f8dd6443bb2075e67331d3152eef43 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Tue, 3 Oct 2017 12:48:03 -0700 Subject: [PATCH 114/170] Remove duplicated method in OpDesc --- paddle/framework/grad_op_desc_maker.h | 12 ++++++------ paddle/framework/op_desc.cc | 18 ------------------ paddle/framework/op_desc.h | 10 ++-------- 3 files changed, 8 insertions(+), 32 deletions(-) diff --git a/paddle/framework/grad_op_desc_maker.h b/paddle/framework/grad_op_desc_maker.h index b4b6d54bf3..e6d63e4b8a 100644 --- a/paddle/framework/grad_op_desc_maker.h +++ b/paddle/framework/grad_op_desc_maker.h @@ -44,12 +44,12 @@ class GradOpDescMakerBase { return ToGradNames(fwd_op_.Output(name)); } - std::vector InputParamNames() const { - return this->fwd_op_.InputParamNames(); + std::vector InputNames() const { + return this->fwd_op_.InputNames(); } - std::vector OutputParamNames() const { - return this->fwd_op_.OutputParamNames(); + std::vector OutputNames() const { + return this->fwd_op_.OutputNames(); } std::vector Input(const std::string& name) const { @@ -96,12 +96,12 @@ class DefaultGradOpDescMaker : public SingleGradOpDescMaker { OpDescBind grad; grad.SetType(this->GradOpType()); - for (auto& input_param : this->InputParamNames()) { + for (auto& input_param : this->InputNames()) { grad.SetInput(input_param, this->Input(input_param)); grad.SetOutput(GradVarName(input_param), this->InputGrad(input_param)); } - for (auto& output_param : this->OutputParamNames()) { + for (auto& output_param : this->OutputNames()) { grad.SetInput(output_param, this->Output(output_param)); grad.SetInput(GradVarName(output_param), this->OutputGrad(output_param)); } diff --git a/paddle/framework/op_desc.cc b/paddle/framework/op_desc.cc index 33a064890c..852f0f1eb8 100644 --- a/paddle/framework/op_desc.cc +++ b/paddle/framework/op_desc.cc @@ -31,15 +31,6 @@ const std::vector &OpDescBind::Input( return it->second; } -std::vector OpDescBind::InputNames() const { - std::vector retv; - retv.reserve(this->inputs_.size()); - for (auto &ipt : this->inputs_) { - retv.push_back(ipt.first); - } - return retv; -} - void OpDescBind::SetInput(const std::string ¶m_name, const std::vector &args) { need_update_ = true; @@ -54,15 +45,6 @@ const std::vector &OpDescBind::Output( return it->second; } -std::vector OpDescBind::OutputNames() const { - std::vector retv; - retv.reserve(this->outputs_.size()); - for (auto &ipt : this->outputs_) { - retv.push_back(ipt.first); - } - return retv; -} - void OpDescBind::SetOutput(const std::string ¶m_name, const std::vector &args) { need_update_ = true; diff --git a/paddle/framework/op_desc.h b/paddle/framework/op_desc.h index 12706b9d71..508bcaa67e 100644 --- a/paddle/framework/op_desc.h +++ b/paddle/framework/op_desc.h @@ -35,15 +35,11 @@ class OpDescBind { const std::vector &Input(const std::string &name) const; - std::vector InputNames() const; - void SetInput(const std::string ¶m_name, const std::vector &args); const std::vector &Output(const std::string &name) const; - std::vector OutputNames() const; - void SetOutput(const std::string ¶m_name, const std::vector &args); @@ -71,10 +67,8 @@ class OpDescBind { // Only be used in C++ void SetAttrMap(const AttributeMap &attr_map); - std::vector InputParamNames() const { return MapKeys(inputs_); } - std::vector OutputParamNames() const { - return MapKeys(outputs_); - } + std::vector InputNames() const { return MapKeys(inputs_); } + std::vector OutputNames() const { return MapKeys(outputs_); } private: template From 495a80a73645bcabe9e392a6b1a2878845f7a234 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Tue, 3 Oct 2017 13:17:10 -0700 Subject: [PATCH 115/170] Update design doc --- doc/design/register_grad_op.md | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/doc/design/register_grad_op.md b/doc/design/register_grad_op.md index 12b04fb271..cdb7a8435b 100644 --- a/doc/design/register_grad_op.md +++ b/doc/design/register_grad_op.md @@ -33,22 +33,45 @@ The mapping relationship between an operator and its gradient operators is a fun ```cpp // (OpDesc) --> vector -using GradOpDescMaker = std::function(const OpDesc&)>; +std::function(const OpDescBind&)>; ``` -The function take a `OpDesc` of the forward operator and return one or many gradient operator descriptions. +The function takes an `OpDescBind` of the forward operator and returns one or many gradient operator descriptions. `OpDescBind` is a C++ wrapper for protobuf message `OpDesc` to manipulate `OpDesc` fast. The `GradOpDescMaker` will be registered in `OpInfo`, to replace `grad_op_type_` field. The `OpInfo` should be ```cpp struct OpInfo { - GradOpDescMaker grad_op_maker_; + std::function(const OpDescBind&)> grad_op_maker_; ... }; ``` The `grad_op_maker_ ` is `nullptr` if the operator does not have associated gradient operators. +We propose a base class called `GradOpDescMakerBase` to let operator developers generate `Gradient Operators` easily. The public interface of that class is + +```cpp +class GradOpDescMakerBase { +public: + GradOpDescMakerBase(const OpDescBind& ); + virtual std::vector operator()()const = 0; +}; +``` + +We can convert `GradOpDescMakerBase` to `std::function(const OpDescBind&)>` by + +```cpp +using GradOpMaker = ...; +std::function(const OpDescBind&)> func; +func = [] (const OpDescBind& fwd_op) { + GradOpMaker maker(fwd_op); + return maker(); +}; +``` + +We can write many helper functions since the `GradOpDescMakerBase` is a class now. The basic helper functions get the variables of `Input`, `Output`, `InputGradient` and `OutputGradient` in the forwarding operator. + We should chagne register macros at the same time. In the current solution, there is no difference between forwarding operators and backward operators. So `REGISTER_OP` just register one operator. If the `REGISTER_OPERATOR ` contains `OpProtoAndCheckerMaker` and `GradOpDescMaker`, we just list them in the same macro. It can be done by a macro contains `__VA_ARGS__`. The user interface should be From 62de57e1ee28bdb349148028d079b4e3192ecb46 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Tue, 3 Oct 2017 14:01:22 -0700 Subject: [PATCH 116/170] Update lod_tensor.md --- paddle/framework/lod_tensor.md | 168 +++++++++++++++++++-------------- 1 file changed, 97 insertions(+), 71 deletions(-) diff --git a/paddle/framework/lod_tensor.md b/paddle/framework/lod_tensor.md index 07bbdf9416..0fa14f3470 100644 --- a/paddle/framework/lod_tensor.md +++ b/paddle/framework/lod_tensor.md @@ -1,147 +1,173 @@ # Design Doc: LoD (Level-of-Detail) Tensor -PaddlePaddle's RNN doesn't require that all instances have the same length. To do so, we introduce an extension to Tensor, namely, LoD Tensor. +As other deep learning systems, PaddlePaddle supports training models from sequence data. Also, like other systems, PaddlePaddle represent a mini-batch of sequences as a Tensor. What is different is that PaddlePaddle doesn't require that all sequences in a mini-batch are of the same length. Thus no need for padding zeros. -## Challenge of Variable-length Inputs +| | TensorFlow | PaddlePaddle | +|-----------------------|------------|--------------| +| RNN | Support | Support | +| recursive RNN | Support | Support | +| padding zeros | Must | No need | +| blob data type | Tensor | LoDTensor | -People usually represent a mini-batch by a Tensor. For example, a mini-batch of 10 images, each of size 32x32, is a 10x32x32 Tensor. So a transformation, T, of all images can be a matrix multiplication of the 10xOx32-dimensional tensor T and the 10x32x32 Tensor. +PaddlePaddle achieves this flexibility by passing through a new data type, *LoD Tensor*, which is a Tensor attached with segmentation index known as *LoD*, between operators. The LoD index doesn't only segments a tensor, but also recursively segments sub-sequences. This document presents the design of LoD and LoDTensor. -Another example is that each mini-batch contains 32 sentences, where each word is a D-dimensional one-hot vector. If all sentences have the same length L, we can represent this mini-batch by a 32xLxD tensor. However, in most cases, sentences have variable lengths, and we will need an index data structure to record these variable lengths. -## LoD as a Solution +## The Challenge: Variable-length Sequences -### Mini-Batch of variable-length sentences +Most deep learning systems represent a mini-batch as a Tensor. For example, a mini-batch of 10 images, each of size 32x32, is a 10x32x32 Tensor. Another example is that each mini-batch contains N sentences, where each word is a D-dimensional one-hot vector. Suppose that all sentences have the same length L, we can represent this mini-batch by a NxLxD tensor. -Let's imagine a mini-batch of 3 variable lengths sentences, containing 3, 1, and 2 words respectively. We can represent it by a (3+1+2)xD tensor plus some index information: +Both examples show that the elements of sequences are usually of the same size. In the first example, all images are 32x32, and in the second one, all words are D-dimensional vectors. It doesn't make sense to allow variable-sized images, as that would require transformations like convolution represented by variable-sized Tensors. + +The real challenge is that in most cases, sentences have variable lengths, and we will need an index data structure to segment the tensor into sequences. Also, sequences might consist of sub-sequences. + +## A Solution: The LoD Index + +Let is visit this challenge from examples. + +### A Mini-Batch of Sentences + +Let's imagine a mini-batch of 3 variable lengths sentences composed by 3, 1, and 2 words respectively. We can represent it by a (3+1+2)xD tensor plus some index information: ``` - 3 3 1 2 ||| | || ``` -Each `|` represents a D-dimensional word vectors. The number 3 on top indicate 3 sentences, and numbers 3, 1, and 2 on the second level represent the number of words in each sentence. +where each `|` represents a D-dimensional word vector. The numbers, 3, 1, and 2, form a 1-level LoD. + +### Recursive Sequences + +Let check another example of a 2-level LoD Tensor. Consider a mini-batch of three articles with 3, 1, and 2 sentences, and each sentence consists of words: + +``` +3 1 2 +3 2 4 1 2 3 +||| || |||| | || ||| +``` -### Mini-Batch of variable-length videos +### A Mini-Batch of Videos -This approach generalizes to the case where elements are not words, but higher dimensional objects, like images. Suppose that a mini-batch contains videos of the same frame size 640x480. If a mini-batch contains 3 videos of 3, 1, and 2 frames respectively. The underlying tensor is of size (3+1+2)x640x480. The index information illustrates as: +LoD Tensor generalizes to the case where elements are higher dimensional objects, like images. Suppose that a mini-batch contains videos of the same frame size 640x480. Here is a mini-batch of 3 videos with 3, 1, and 2 frames respectively. ``` - 3 3 1 2 口口口 口 口口 ``` -where each `口` represents an image. +The underlying tensor is of size (3+1+2)x640x480, and each `口` represents a 640x480 image. -### Mini-Batch of fixed-size images +### A Mini-Batch of Images -Let's get back to a typical example, image classification, where each mini-batch has M fixed-sized images. The LoD Tensor representation is +In traditional cases like a mini-batch with N fixed-sized images, the LoD Tensor representation is as ``` - M 1 1 1 1 1 口口口口 ... 口 ``` -The many 1's on the second level seem duplicated. For this particular case of 2 levels and the second level always have length 1, we can ignore the LoD index. - -### Design and summarization - -In summary, as long as that the essential elements (words or images) have the same size, we can represent mini-batches by a LoD Tensor: +It doesn't loss anything to ignore the many 1's in the index and to consider this LoD Tensor a usual Tensor: -- The underlying tensor has size LxD1xD2x..., where D1xD2... is the size of the essential elements, and -- The first dimension size L has an additonal property -- a LoD index as a nested vector: +``` +口口口口 ... 口 +``` - ```c++ - typedef std::vector> LoD; - ``` +### Model Parameters -- The LoD index is not necessary when there are only two levels and all elements of the second level have length 1. +A model parameter is just a usual Tensor, which, just like the above example, is a **0-level LoD Tensor**. -## Slicing of LoD Tensor +## The LoD Tensor -Consider that we have a network with three levels of RNN: the top level one handles articles, the second level one handles sentences, and the basic level one handles words. This network requires that mini-batches represented by 3 level LoD Tensor, for example, +Let us revisit above example of the 2-level LoD Tensor ``` - 3 3 1 2 3 2 4 1 2 3 ||| || |||| | || ||| ``` -To allow each level of RNN to handle its input, we define **the slicing of a LoD Tensor is defined as getting the j-th sequence on level i, or the -slice** +It is indeed a tree, where leaves are elementary sequences identified by **branches**. + +For example, the third sentence in above example is identified by branch <0,2>, where 0 indicates the first article with length 3, and 2 indicates the third sentence in this article with length 4. + +### The LoD Index -For example, the <2,1>-slice of above slice is +We can save the LoD index in above example ``` -2 -|| +3 1 2 +3 2 4 1 2 3 ``` -and the <1,2>-slice of above example is +in a not-full 2D matrix: +```c++ +typedef std::vector > LoD; ``` -2 -2 3 -|| ||| -``` -Let's go on slicing this slice. Its <1,1>-slice is +where + +- `LoD.size()` is the number of levels, or the maximum length of branches, +- `LoD[i][j]` is the length of the j-th segment at the i-th level. + +## The Offset Representation + +To quickly access elementary sequences, we adopt an offset representation -- instead of saving the lengths, we save the beginning and ending elements of sequences. + +In the above example, we accumulate the length of elementary sequences: ``` -1 -1 -| +3 2 4 1 2 3 ``` -### The Slicing Algorithm +into offsets -The algorithm, with over-simplified data structure, is defined as +``` +0 3 5 9 10 12 15 + = = = = = = + 3 2+3 4+5 1+9 2+10 3+12 +``` -```c++ -typedef std::vector> LoD; +so we know that the first sentence is from word 0 to word 3, and the second sentence from work 3 to word 5. -struct LoDTensor { - LoD lod_; - float* tensor_; -}; +Similarly, lengths in the top level LoD -LoDTensor Slice(const LoDTensor& lodt, int level, int sequence); +``` +3 1 2 ``` -Let us revisit the example above +is transformed into offsets of elements/words: ``` - 3 -3 1 2 -3 2 4 1 2 3 -||| || |||| | || ||| +0 9 10 15 + = = = + 3+2+4 1+9 2+3+10 ``` -Suppose that we want to retrieve the <1,2>-slice +so we can tell that the first article is from word 0 to word 9, and the second article is from word 9 to word 10. + +The complete offset representation is as follows: ``` -2 -2 3 -|| ||| +0 9 10 15 +0 3 5 9 10 12 15 +||| || |||| | || ||| ``` -we will need to find out the starting position of this slice by summing over all leaf nodes in `LoD` to the left of the slice, i.e., 3 + 2 + 4 + 1 = 10. +## Slicing of LoD Tensors + +When we use the above 2-level LoD Tensor as the input to a nested-RNN, we need to retrieve certain sequences. Here we define the sequence identified by branch as the **-slice**. -To avoid the traversal of the LoD tree at slicing time, we can do it at the construction time -- instead of saving the lengths of the next level in the LoD tree, we can save the starting offset of the next level. For example, above LoD Tensor can be transformed into +For example, the <2>-slice of above example is ``` - 0 -0 9 10 -0 3 5 9 10 12 -||| || |||| | || ||| +10 15 +10 12 15 + || ||| ``` -We don't really need the 0 on top, so the LoD Tensor could be +and the <2,0>-slice of above slice is ``` -0 9 10 -0 3 5 9 10 12 -||| || |||| | || ||| +10 12 + || ``` From 9935fdd3dd92cf9930f88b070090925d2909ed1a Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Tue, 3 Oct 2017 14:14:42 -0700 Subject: [PATCH 117/170] Update --- paddle/framework/backward.cc | 57 ++++++++++++++++++++++++++++++++--- paddle/framework/block_desc.h | 4 +++ paddle/framework/op_desc.cc | 16 ++++++++++ paddle/framework/op_desc.h | 4 +++ 4 files changed, 77 insertions(+), 4 deletions(-) diff --git a/paddle/framework/backward.cc b/paddle/framework/backward.cc index 1b4c5c025e..0f65478ef8 100644 --- a/paddle/framework/backward.cc +++ b/paddle/framework/backward.cc @@ -235,14 +235,17 @@ static bool AllGradInSet(const std::vector& names, } std::vector CreatBackwardOps( - const OpDescBind& op_desc, unordered_map& no_grad_vars) { + const std::unique_ptr& op_desc_ptr, + unordered_map& no_grad_vars) { + const OpDescBind& op_desc = *op_desc_ptr; std::vector grad_op_descs; // All input gradients of forwarding operator do not need to calculat. - if (AllGradInSet(op_desc_.InputNames(), kGradVarSuffix, no_grad_vars)) { + if (AllGradInSet(op_desc_.InputArgumentNames(), kGradVarSuffix, + no_grad_vars)) { return grad_op_descs; // empty vector } // All output gradients of forwarding operator do not need to calculate. - const std::vector& outputs = op_desc_.OutputNames(); + const std::vector& outputs = op_desc_.OutputArugumentNames(); if (AllGradInSet(outputs, kGradVarSuffix, no_grad_vars)) { for (const std::string& name : outputs) { no_grad_vars.insert(GradVarName(name)); @@ -254,7 +257,7 @@ std::vector CreatBackwardOps( std::vector fill_zeros_ops; for (OpDescBind& desc : grad_op_descs) { - for (const std::string& in_name : desc.InputNames()) { + for (const std::string& in_name : desc.InputArgumentNames()) { if (no_grad_vars.count(in_name)) { std::string prefix = in_name.substr( 0, in_name.size() - sizeof(kGradVarSuffix) / sizeof(char) + 1); @@ -278,5 +281,51 @@ std::vector CreatBackwardOps( return grad_op_descs; } +void AppendBackwardOps(BlockDescBind& block_desc, + const std::unordered_set& no_grad_vars) { + std::unordered_map> dup_out_ops; + size_t grad_desc_idx = 0; + std::deque> op_descs = block_desc.ops_; + std::vector> grad_op_descs; + for (auto it = op_descs.rbegin(); it != op_descs.rend(); ++it) { + std::vector op_grads = CreatBackwardOps(*it, no_grad_vars); + for (const OpDescBind& desc : op_grads) { + for (const std::string& out_name : desc.OutputArugumentNames()) { + dup_out_ops[out_name].emplace_back(grad_desc_idx); + } + ++grad_desc_idx; + } + grad_op_descs.insert(grad_op_descs.end(), op_grads.begin(), op_grads.end()); + } + // Check whether some variables are written more than once + std::list> pending_sum_ops; + for (const auto& dup : dup_out_ops) { + const std::string& out_name = dup.first; + const std::vector dup_op = dup.second; + if (out_name != kEmptyVarName && dup_op.size() > 1) { + std::vector sum_op_inputs; + for (size_t i = 0; i < dup_op.size(); ++i) { + std::string new_name = out_name + "@RENAME@" + std::to_string(i); + grad_op_descs[dup_op[i]].Rename(out_name, new_name); + sum_op_inputs.emplace_back(new_name); + } + pending_sum_ops.push_back( + {dup_op.back(), + OpDescBind( + {"sum", {{"X", {sum_op_inputs}}}, {{"Out", {out_name}}}, {}})}); + } + } + pending_sum_ops.sort( + [](const std::pair& a, + const std::pair& b) { return a.first > b.first; }); + for (auto& p : pending_sum_ops) { + grad_op_descs.insert(grad_op_descs.begin() + p.first + 1, + std::move(p.second)); + } + // Append grad_op_descs to BlockDescBind::ops_ + for () { + } +} + } // namespace framework } // namespace paddle diff --git a/paddle/framework/block_desc.h b/paddle/framework/block_desc.h index 59513ede33..a171dfef30 100644 --- a/paddle/framework/block_desc.h +++ b/paddle/framework/block_desc.h @@ -32,6 +32,10 @@ class ProgramDescBind; class BlockDescBind { public: + friend void AppendBackwardOps( + BlockDescBind &block_desc, + const std::unordered_set &no_grad_vars); + BlockDescBind(ProgramDescBind *prog, BlockDesc *desc) : prog_(prog), desc_(desc), need_update_(false) {} diff --git a/paddle/framework/op_desc.cc b/paddle/framework/op_desc.cc index f2e0c14fbd..e6c0cdacd9 100644 --- a/paddle/framework/op_desc.cc +++ b/paddle/framework/op_desc.cc @@ -49,6 +49,14 @@ std::vector OpDescBind::InputNames() const { return retv; } +std::vector InputArgumentNames() const { + std::vector retv; + for (auto &ipt : this->inputs_) { + retv.insert(retv.end(), ipt.second.begin(), ipt.second.end()); + } + return retv; +} + void OpDescBind::SetInput(const std::string ¶m_name, const std::vector &args) { need_update_ = true; @@ -72,6 +80,14 @@ std::vector OpDescBind::OutputNames() const { return retv; } +std::vector OutputArgumentNames() const { + std::vector retv; + for (auto &ipt : this->outputs_) { + retv.insert(retv.end(), ipt.second.begin(), ipt.second.end()); + } + return retv; +} + void OpDescBind::SetOutput(const std::string ¶m_name, const std::vector &args) { need_update_ = true; diff --git a/paddle/framework/op_desc.h b/paddle/framework/op_desc.h index 2280654812..e30c58632e 100644 --- a/paddle/framework/op_desc.h +++ b/paddle/framework/op_desc.h @@ -42,6 +42,8 @@ class OpDescBind { std::vector InputNames() const; + std::vector InputArgumentNames() const; + void SetInput(const std::string ¶m_name, const std::vector &args); @@ -49,6 +51,8 @@ class OpDescBind { std::vector OutputNames() const; + std::vector OutputArgumentNames() const; + void SetOutput(const std::string ¶m_name, const std::vector &args); From 48a9ab4a0896b3102637fb7606b27bbf6b097bc3 Mon Sep 17 00:00:00 2001 From: Markus Kliegl Date: Tue, 3 Oct 2017 14:21:42 -0700 Subject: [PATCH 118/170] minor language fixes --- paddle/framework/lod_tensor.md | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/paddle/framework/lod_tensor.md b/paddle/framework/lod_tensor.md index 0fa14f3470..597bc48cf3 100644 --- a/paddle/framework/lod_tensor.md +++ b/paddle/framework/lod_tensor.md @@ -1,6 +1,6 @@ # Design Doc: LoD (Level-of-Detail) Tensor -As other deep learning systems, PaddlePaddle supports training models from sequence data. Also, like other systems, PaddlePaddle represent a mini-batch of sequences as a Tensor. What is different is that PaddlePaddle doesn't require that all sequences in a mini-batch are of the same length. Thus no need for padding zeros. +Like other deep learning systems, PaddlePaddle supports training models from sequence data. Also, like other systems, PaddlePaddle represent a mini-batch of sequences as a Tensor. What is different is that PaddlePaddle doesn't require all sequences in a mini-batch to be of the same length. Thus no need for padding zeros. | | TensorFlow | PaddlePaddle | |-----------------------|------------|--------------| @@ -9,24 +9,24 @@ As other deep learning systems, PaddlePaddle supports training models from seque | padding zeros | Must | No need | | blob data type | Tensor | LoDTensor | -PaddlePaddle achieves this flexibility by passing through a new data type, *LoD Tensor*, which is a Tensor attached with segmentation index known as *LoD*, between operators. The LoD index doesn't only segments a tensor, but also recursively segments sub-sequences. This document presents the design of LoD and LoDTensor. +PaddlePaddle achieves this flexibility by passing through a new data type, *LoD Tensor*, which is a Tensor attached with segmentation index known as *LoD*, between operators. The LoD index doesn't only segment a tensor, but also recursively segments sub-sequences. This document presents the design of LoD and LoDTensor. ## The Challenge: Variable-length Sequences Most deep learning systems represent a mini-batch as a Tensor. For example, a mini-batch of 10 images, each of size 32x32, is a 10x32x32 Tensor. Another example is that each mini-batch contains N sentences, where each word is a D-dimensional one-hot vector. Suppose that all sentences have the same length L, we can represent this mini-batch by a NxLxD tensor. -Both examples show that the elements of sequences are usually of the same size. In the first example, all images are 32x32, and in the second one, all words are D-dimensional vectors. It doesn't make sense to allow variable-sized images, as that would require transformations like convolution represented by variable-sized Tensors. +Both examples show that the elements of sequences are usually of the same size. In the first example, all images are 32x32, and in the second one, all words are D-dimensional vectors. It doesn't make sense to allow variable-sized images, as that would require transformations like convolution to handle variable-sized Tensors. The real challenge is that in most cases, sentences have variable lengths, and we will need an index data structure to segment the tensor into sequences. Also, sequences might consist of sub-sequences. ## A Solution: The LoD Index -Let is visit this challenge from examples. +To understand our solution, it is best to look at some examples. ### A Mini-Batch of Sentences -Let's imagine a mini-batch of 3 variable lengths sentences composed by 3, 1, and 2 words respectively. We can represent it by a (3+1+2)xD tensor plus some index information: +Let's imagine a mini-batch of 3 variable lengths sentences composed of 3, 1, and 2 words, respectively. We can represent the mini-batch by a (3+1+2)xD tensor plus some index information: ``` 3 1 2 @@ -37,7 +37,7 @@ where each `|` represents a D-dimensional word vector. The numbers, 3, 1, and 2 ### Recursive Sequences -Let check another example of a 2-level LoD Tensor. Consider a mini-batch of three articles with 3, 1, and 2 sentences, and each sentence consists of words: +Let check another example of a 2-level LoD Tensor. Consider a mini-batch of three articles with 3, 1, and 2 sentences, and each sentence consists of a variable number of words: ``` 3 1 2 @@ -47,7 +47,7 @@ Let check another example of a 2-level LoD Tensor. Consider a mini-batch of thr ### A Mini-Batch of Videos -LoD Tensor generalizes to the case where elements are higher dimensional objects, like images. Suppose that a mini-batch contains videos of the same frame size 640x480. Here is a mini-batch of 3 videos with 3, 1, and 2 frames respectively. +LoD tensors generalize to the case where elements are higher dimensional objects, like images. Suppose that a mini-batch contains videos of the same frame size 640x480. Here is a mini-batch of 3 videos with 3, 1, and 2 frames, respectively. ``` 3 1 2 @@ -65,7 +65,7 @@ In traditional cases like a mini-batch with N fixed-sized images, the LoD Tenso 口口口口 ... 口 ``` -It doesn't loss anything to ignore the many 1's in the index and to consider this LoD Tensor a usual Tensor: +In this case, we don't lose any information by ignoring the many 1's in the index and simply considering this LoD Tensor as a usual Tensor: ``` 口口口口 ... 口 @@ -91,7 +91,7 @@ For example, the third sentence in above example is identified by branch <0,2>, ### The LoD Index -We can save the LoD index in above example +We can save the LoD index in the above example ``` 3 1 2 @@ -129,13 +129,13 @@ into offsets so we know that the first sentence is from word 0 to word 3, and the second sentence from work 3 to word 5. -Similarly, lengths in the top level LoD +Similarly, the lengths in the top level LoD ``` 3 1 2 ``` -is transformed into offsets of elements/words: +are transformed into offsets of elements/words as follows: ``` 0 9 10 15 @@ -148,9 +148,9 @@ so we can tell that the first article is from word 0 to word 9, and the second a The complete offset representation is as follows: ``` -0 9 10 15 -0 3 5 9 10 12 15 -||| || |||| | || ||| +0 9 10 15 +0 3 5 9 10 12 15 + ||| || |||| | || ||| ``` ## Slicing of LoD Tensors From b2806135a53cbe85fbc764375d9cecc2596ab4be Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Tue, 3 Oct 2017 14:41:21 -0700 Subject: [PATCH 119/170] Change Interface to unique_ptr --- doc/design/register_grad_op.md | 6 +++--- paddle/framework/grad_op_desc_maker.h | 28 +++++++++++++++------------ paddle/framework/op_info.h | 2 +- paddle/framework/type_defs.h | 4 ++++ 4 files changed, 24 insertions(+), 16 deletions(-) diff --git a/doc/design/register_grad_op.md b/doc/design/register_grad_op.md index cdb7a8435b..3cf8a59446 100644 --- a/doc/design/register_grad_op.md +++ b/doc/design/register_grad_op.md @@ -42,7 +42,7 @@ The `GradOpDescMaker` will be registered in `OpInfo`, to replace `grad_op_type_` ```cpp struct OpInfo { - std::function(const OpDescBind&)> grad_op_maker_; + std::function>(const OpDescBind&)> grad_op_maker_; ... }; ``` @@ -55,11 +55,11 @@ We propose a base class called `GradOpDescMakerBase` to let operator developers class GradOpDescMakerBase { public: GradOpDescMakerBase(const OpDescBind& ); - virtual std::vector operator()()const = 0; + virtual std::vector> operator()()const = 0; }; ``` -We can convert `GradOpDescMakerBase` to `std::function(const OpDescBind&)>` by +We can convert `GradOpDescMakerBase` to `std::function>(const OpDescBind&)>` by ```cpp using GradOpMaker = ...; diff --git a/paddle/framework/grad_op_desc_maker.h b/paddle/framework/grad_op_desc_maker.h index e6d63e4b8a..e9ae6e2206 100644 --- a/paddle/framework/grad_op_desc_maker.h +++ b/paddle/framework/grad_op_desc_maker.h @@ -24,7 +24,7 @@ class GradOpDescMakerBase { explicit GradOpDescMakerBase(const OpDescBind& fwd_op) : fwd_op_(fwd_op) {} virtual ~GradOpDescMakerBase() = default; - virtual std::vector operator()() const = 0; + virtual std::vector> operator()() const = 0; protected: static std::vector ToGradNames( @@ -81,10 +81,14 @@ class SingleGradOpDescMaker : public GradOpDescMakerBase { public: using GradOpDescMakerBase::GradOpDescMakerBase; - std::vector operator()() const { return {this->Apply()}; } + std::vector> operator()() const { + std::vector> retv; + retv.emplace_back(this->Apply()); + return retv; + } protected: - virtual OpDescBind Apply() const = 0; + virtual std::unique_ptr Apply() const = 0; }; class DefaultGradOpDescMaker : public SingleGradOpDescMaker { @@ -92,23 +96,23 @@ class DefaultGradOpDescMaker : public SingleGradOpDescMaker { using SingleGradOpDescMaker::SingleGradOpDescMaker; protected: - virtual OpDescBind Apply() const { - OpDescBind grad; - grad.SetType(this->GradOpType()); + virtual std::unique_ptr Apply() const { + auto* grad = new OpDescBind(); + grad->SetType(this->GradOpType()); for (auto& input_param : this->InputNames()) { - grad.SetInput(input_param, this->Input(input_param)); - grad.SetOutput(GradVarName(input_param), this->InputGrad(input_param)); + grad->SetInput(input_param, this->Input(input_param)); + grad->SetOutput(GradVarName(input_param), this->InputGrad(input_param)); } for (auto& output_param : this->OutputNames()) { - grad.SetInput(output_param, this->Output(output_param)); - grad.SetInput(GradVarName(output_param), this->OutputGrad(output_param)); + grad->SetInput(output_param, this->Output(output_param)); + grad->SetInput(GradVarName(output_param), this->OutputGrad(output_param)); } - grad.SetAttrMap(this->Attrs()); + grad->SetAttrMap(this->Attrs()); - return grad; + return std::unique_ptr(grad); } virtual std::string GradOpType() const { diff --git a/paddle/framework/op_info.h b/paddle/framework/op_info.h index 806a960018..8b7882485f 100644 --- a/paddle/framework/op_info.h +++ b/paddle/framework/op_info.h @@ -28,7 +28,7 @@ namespace framework { struct OpInfo { OpCreator creator_; std::string grad_op_type_; - std::function(const OpDescBind&)> grad_op_maker_; + GradOpMakerFN grad_op_maker_; OpProto* proto_{nullptr}; OpAttrChecker* checker_{nullptr}; diff --git a/paddle/framework/type_defs.h b/paddle/framework/type_defs.h index dec5066f1e..a5b9472213 100644 --- a/paddle/framework/type_defs.h +++ b/paddle/framework/type_defs.h @@ -20,6 +20,7 @@ namespace paddle { namespace framework { class OperatorBase; +class OpDescBind; using VariableNameMap = std::map>; // The order should be as same as framework.proto @@ -34,5 +35,8 @@ using OpCreator = std::function; +using GradOpMakerFN = + std::function>(const OpDescBind&)>; + } // namespace framework } // namespace paddle From e47770bd27a20b2fa9bf2754d16b0e71008185e5 Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Tue, 3 Oct 2017 15:30:21 -0700 Subject: [PATCH 120/170] Update --- paddle/framework/backward.cc | 85 +++++++++++++++++---------------- paddle/framework/backward.h | 2 +- paddle/framework/op_desc.cc | 13 ++--- paddle/framework/op_registry.cc | 3 +- paddle/framework/op_registry.h | 3 +- 5 files changed, 57 insertions(+), 49 deletions(-) diff --git a/paddle/framework/backward.cc b/paddle/framework/backward.cc index 0f65478ef8..b4eb89e2d7 100644 --- a/paddle/framework/backward.cc +++ b/paddle/framework/backward.cc @@ -234,18 +234,17 @@ static bool AllGradInSet(const std::vector& names, return true; } -std::vector CreatBackwardOps( - const std::unique_ptr& op_desc_ptr, - unordered_map& no_grad_vars) { - const OpDescBind& op_desc = *op_desc_ptr; - std::vector grad_op_descs; +std::vector> MakeGradOpDescs( + const std::unique_ptr& op_desc, + unordered_set& no_grad_vars) { + std::vector> grad_op_descs; // All input gradients of forwarding operator do not need to calculat. - if (AllGradInSet(op_desc_.InputArgumentNames(), kGradVarSuffix, + if (AllGradInSet(op_desc->InputArgumentNames(), kGradVarSuffix, no_grad_vars)) { return grad_op_descs; // empty vector } // All output gradients of forwarding operator do not need to calculate. - const std::vector& outputs = op_desc_.OutputArugumentNames(); + const std::vector& outputs = op_desc->OutputArugumentNames(); if (AllGradInSet(outputs, kGradVarSuffix, no_grad_vars)) { for (const std::string& name : outputs) { no_grad_vars.insert(GradVarName(name)); @@ -255,50 +254,54 @@ std::vector CreatBackwardOps( grad_op_descs = OpRegistry::CreateGradOpDescs(op_desc); - std::vector fill_zeros_ops; - for (OpDescBind& desc : grad_op_descs) { - for (const std::string& in_name : desc.InputArgumentNames()) { + std::list> pending_fill_zeros_ops; + for (auto& desc : grad_op_descs) { + for (const std::string& in_name : desc->InputArgumentNames()) { if (no_grad_vars.count(in_name)) { std::string prefix = in_name.substr( 0, in_name.size() - sizeof(kGradVarSuffix) / sizeof(char) + 1); std::string new_name = prefix + kZeroVarSuffix; - desc.Rename(in_name, new_name); - OpDescBind op_desc_bind( - {"fill_zeros_like", {{"X", {prefix}}}, {{"Y", {new_name}}}, {}}); - fill_zeros_ops.push_back(op_desc_bind); + desc->Rename(in_name, new_name); + OpDescBind* fill_zeros_op = new OpDescBind( + "fill_zeros_like", {{"X", {prefix}}}, {{"Y", {new_name}}}, {}); + pending_fill_zeros_ops.push_back({fill_zeros_op}); } } - for (const std::string& out_name : desc.OutputName()) { + for (const std::string& out_name : desc->OutputArgumentName()) { if (no_grad_vars.count(out_name)) { - desc.Rename(out_name, kEmptyVarName); + desc->Rename(out_name, kEmptyVarName); } } } - grad_op_descs.insert(grad_op_descs.begin(), fill_zeros_ops.begin(), - fill_zeros_ops.end()); + grad_op_descs.insert(std::begin(grad_op_descs), + std::begin(pending_fill_zeros_ops), + std::end(pending_fill_zeros_ops)); // TODO (fengjiayi): RNN op return grad_op_descs; } -void AppendBackwardOps(BlockDescBind& block_desc, - const std::unordered_set& no_grad_vars) { +void AppendBackwardOpDescs( + BlockDescBind& block_desc, + const std::unordered_set& no_grad_vars) { std::unordered_map> dup_out_ops; size_t grad_desc_idx = 0; - std::deque> op_descs = block_desc.ops_; - std::vector> grad_op_descs; - for (auto it = op_descs.rbegin(); it != op_descs.rend(); ++it) { - std::vector op_grads = CreatBackwardOps(*it, no_grad_vars); - for (const OpDescBind& desc : op_grads) { - for (const std::string& out_name : desc.OutputArugumentNames()) { + std::deque> block_op_descs = block_desc.ops_; + std::vector> backward_descs; + for (auto it = block_op_descs.rbegin(); it != block_op_descs.rend(); ++it) { + std::vector> op_grads = + MakeGradOpDescs(*it, no_grad_vars); + for (const auto& desc : op_grads) { + for (const std::string& out_name : desc->OutputArugumentNames()) { dup_out_ops[out_name].emplace_back(grad_desc_idx); } ++grad_desc_idx; } - grad_op_descs.insert(grad_op_descs.end(), op_grads.begin(), op_grads.end()); + backward_descs.insert(backward_descs.end(), op_grads.begin(), + op_grads.end()); } // Check whether some variables are written more than once - std::list> pending_sum_ops; + std::list>> pending_sum_ops; for (const auto& dup : dup_out_ops) { const std::string& out_name = dup.first; const std::vector dup_op = dup.second; @@ -306,25 +309,27 @@ void AppendBackwardOps(BlockDescBind& block_desc, std::vector sum_op_inputs; for (size_t i = 0; i < dup_op.size(); ++i) { std::string new_name = out_name + "@RENAME@" + std::to_string(i); - grad_op_descs[dup_op[i]].Rename(out_name, new_name); + backward_descs[dup_op[i]]->Rename(out_name, new_name); sum_op_inputs.emplace_back(new_name); } - pending_sum_ops.push_back( - {dup_op.back(), - OpDescBind( - {"sum", {{"X", {sum_op_inputs}}}, {{"Out", {out_name}}}, {}})}); + OpDescBind* sum_op = new OpDescBind("sum", {{"X", sum_op_inputs}}, + {{"Out", {out_name}}}, {}); + pending_sum_ops.push_back({dup_op.back(), {sum_op}}); } } pending_sum_ops.sort( - [](const std::pair& a, - const std::pair& b) { return a.first > b.first; }); + [](const std::pair>& a, + const std::pair>& b) { + return a.first > b.first; + }); for (auto& p : pending_sum_ops) { - grad_op_descs.insert(grad_op_descs.begin() + p.first + 1, - std::move(p.second)); - } - // Append grad_op_descs to BlockDescBind::ops_ - for () { + backward_descs.insert(backward_descs.begin() + p.first + 1, + std::move(p.second)); } + // Append backward_descs to BlockDescBind::ops_ + block_op_descs.insert(std::end(block_op_descs), std::begin(backward_descs), + std::end(backward_descs)); + return; } } // namespace framework diff --git a/paddle/framework/backward.h b/paddle/framework/backward.h index 6aeddafb41..fb496c34c7 100644 --- a/paddle/framework/backward.h +++ b/paddle/framework/backward.h @@ -24,7 +24,7 @@ extern std::unique_ptr Backward( const OperatorBase& forwardOp, const std::unordered_set& no_grad_vars); -extern void AppendBackwardOps( +extern void AppendBackwardOpDescs( BlockDescBind& block_desc, const std::unordered_set& no_grad_vars); diff --git a/paddle/framework/op_desc.cc b/paddle/framework/op_desc.cc index e6c0cdacd9..2c6aec717b 100644 --- a/paddle/framework/op_desc.cc +++ b/paddle/framework/op_desc.cc @@ -49,7 +49,7 @@ std::vector OpDescBind::InputNames() const { return retv; } -std::vector InputArgumentNames() const { +std::vector OpDescBind::InputArgumentNames() const { std::vector retv; for (auto &ipt : this->inputs_) { retv.insert(retv.end(), ipt.second.begin(), ipt.second.end()); @@ -80,7 +80,7 @@ std::vector OpDescBind::OutputNames() const { return retv; } -std::vector OutputArgumentNames() const { +std::vector OpDescBind::OutputArgumentNames() const { std::vector retv; for (auto &ipt : this->outputs_) { retv.insert(retv.end(), ipt.second.begin(), ipt.second.end()); @@ -137,12 +137,13 @@ const std::unordered_map &OpDescBind::GetAttrMap() return attrs_; } -void Rename(const std::string &old_name, const std::string &new_name) { - for (std : string &input : inputs_) { +void OpDescBind::Rename(const std::string &old_name, + const std::string &new_name) { + for (auto &input : inputs_) { std::replace(input.second.begin(), input.second.end(), old_name, new_name); } - for (std::string &output : outputs_) { - std::repalce(output.second.begin(), output.second.end(), old_name, + for (auto &output : outputs_) { + std::replace(output.second.begin(), output.second.end(), old_name, new_name); } need_update_ = true; diff --git a/paddle/framework/op_registry.cc b/paddle/framework/op_registry.cc index fe3228ce5b..d8851a8b42 100644 --- a/paddle/framework/op_registry.cc +++ b/paddle/framework/op_registry.cc @@ -57,7 +57,8 @@ std::unique_ptr OpRegistry::CreateGradOp(const OperatorBase& op) { return std::unique_ptr(BuildGradOp(&op)); } -static std::vector CreateGradOpDescs(const OpDescBind& op_desc) { +static std::vector> OpRegistry::CreateGradOpDescs( + const OpDescBind& op_desc) { auto& info = OpInfoMap::Instance().Get(op_desc.Type()); return info.grad_op_maker_(op_desc); } diff --git a/paddle/framework/op_registry.h b/paddle/framework/op_registry.h index c80b6e9630..e334cd592a 100644 --- a/paddle/framework/op_registry.h +++ b/paddle/framework/op_registry.h @@ -69,7 +69,8 @@ class OpRegistry { static std::unique_ptr CreateGradOp(const OperatorBase& op); - static std::vector CreateGradOpDescs(const OpDescBind& op_desc); + static std::vector> CreateGradOpDescs( + const OpDescBind& op_desc); }; class Registrar { From e08367c80678804fc388004ba2ab72f754bc1143 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Tue, 3 Oct 2017 15:53:42 -0700 Subject: [PATCH 121/170] Add few blank lines --- paddle/framework/lod_tensor.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/paddle/framework/lod_tensor.md b/paddle/framework/lod_tensor.md index 597bc48cf3..d147f1c425 100644 --- a/paddle/framework/lod_tensor.md +++ b/paddle/framework/lod_tensor.md @@ -20,6 +20,7 @@ Both examples show that the elements of sequences are usually of the same size. The real challenge is that in most cases, sentences have variable lengths, and we will need an index data structure to segment the tensor into sequences. Also, sequences might consist of sub-sequences. + ## A Solution: The LoD Index To understand our solution, it is best to look at some examples. @@ -75,6 +76,7 @@ In this case, we don't lose any information by ignoring the many 1's in the inde A model parameter is just a usual Tensor, which, just like the above example, is a **0-level LoD Tensor**. + ## The LoD Tensor Let us revisit above example of the 2-level LoD Tensor From 7e4648338adbaa6c36f43706771901fc9158447a Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Tue, 3 Oct 2017 16:13:35 -0700 Subject: [PATCH 122/170] Fix bug --- paddle/framework/op_registry.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/framework/op_registry.cc b/paddle/framework/op_registry.cc index d8851a8b42..b28eb64af5 100644 --- a/paddle/framework/op_registry.cc +++ b/paddle/framework/op_registry.cc @@ -57,7 +57,7 @@ std::unique_ptr OpRegistry::CreateGradOp(const OperatorBase& op) { return std::unique_ptr(BuildGradOp(&op)); } -static std::vector> OpRegistry::CreateGradOpDescs( +std::vector> OpRegistry::CreateGradOpDescs( const OpDescBind& op_desc) { auto& info = OpInfoMap::Instance().Get(op_desc.Type()); return info.grad_op_maker_(op_desc); From 703321e2be91d8e70e6578fb3c91f76607f2e587 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Tue, 3 Oct 2017 16:17:49 -0700 Subject: [PATCH 123/170] Fix CI --- paddle/framework/op_desc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/framework/op_desc.h b/paddle/framework/op_desc.h index 508bcaa67e..397393f796 100644 --- a/paddle/framework/op_desc.h +++ b/paddle/framework/op_desc.h @@ -76,7 +76,7 @@ class OpDescBind { std::vector ret_val; ret_val.reserve(map.size()); std::transform( - map.begin(), map.end(), ret_val.begin(), + map.begin(), map.end(), std::back_inserter(ret_val), [](const typename MapType::value_type &pair) { return pair.first; }); return ret_val; } From b0d2235834cd1a94c39c1e937f95c58bd7319abc Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Tue, 3 Oct 2017 16:24:24 -0700 Subject: [PATCH 124/170] Bug fix --- paddle/framework/backward.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/framework/backward.cc b/paddle/framework/backward.cc index 2c13ddd8d0..89583ade95 100644 --- a/paddle/framework/backward.cc +++ b/paddle/framework/backward.cc @@ -147,7 +147,7 @@ static std::unique_ptr BackwardRecursive( for (size_t output_idx = 0; output_idx < dup_outputs.size() - 1; ++output_idx) { auto insert_add_x = dup_outputs[output_idx]; - auto insert_add_y = dup_outputs[output_idx]; + auto insert_add_y = dup_outputs[output_idx + 1]; auto insert_add_out = name + "@SHARED@" + std::to_string(output_idx); // first add op inserted if (output_idx == dup_outputs.size() - 2) { From f4491fa46d1583caa7f007a581995435a32f8dab Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Tue, 3 Oct 2017 16:34:21 -0700 Subject: [PATCH 125/170] Fix bug --- paddle/framework/backward.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/paddle/framework/backward.cc b/paddle/framework/backward.cc index 89583ade95..c0188c0e55 100644 --- a/paddle/framework/backward.cc +++ b/paddle/framework/backward.cc @@ -158,9 +158,8 @@ static std::unique_ptr BackwardRecursive( } insert_position.push_back( {dup_op.back(), - OpRegistry::CreateOp( - "sum", {{"X", {insert_add_x}}, {"X", {insert_add_y}}}, - {{"Out", {insert_add_out}}}, {})}); + OpRegistry::CreateOp("sum", {{"X", {insert_add_x, insert_add_y}}}, + {{"Out", {insert_add_out}}}, {})}); } } @@ -200,7 +199,8 @@ static std::unique_ptr BackwardRecursive( // process recurrent gradient op as a special operator. if (forwardOp.Type() == "recurrent") { - // NOTE clean up cycle call somewhere (RNN's stepnet constains itself), or + // NOTE clean up cycle call somewhere (RNN's stepnet constains itself), + // or // this will result in infinite loop. const auto& rnnop = *static_cast(&forwardOp); From 81fc7774ec02e23163014e66eeddeea43e5dd703 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Tue, 3 Oct 2017 16:59:32 -0700 Subject: [PATCH 126/170] optimize infershape context --- paddle/framework/operator.h | 64 ++++++++++++++++++++----------------- 1 file changed, 34 insertions(+), 30 deletions(-) diff --git a/paddle/framework/operator.h b/paddle/framework/operator.h index 5bb5c8e2f9..99f721cc67 100644 --- a/paddle/framework/operator.h +++ b/paddle/framework/operator.h @@ -323,74 +323,76 @@ class CompileTimeInferShapeContext : public InferShapeContextBase { CompileTimeInferShapeContext(const OpDescBind& op, const BlockDescBind& block) : op_(op), block_(block) {} - bool HasInput(const std::string& name) const { + bool HasInput(const std::string& name) const override { const std::vector& input_names = op_.Input(name); PADDLE_ENFORCE_EQ(input_names.size(), 1UL, "Inputs(%s) length is not 1", name); return block_.HasVar(input_names[0]); } - bool HasOutput(const std::string& name) const { + bool HasOutput(const std::string& name) const override { const std::vector& output_names = op_.Output(name); PADDLE_ENFORCE_EQ(output_names.size(), 1UL, "Outputs(%s) length is not 1", name); return block_.HasVar(output_names[0]); } - bool HasInputs(const std::string& name) const { + bool HasInputs(const std::string& name) const override { const std::vector& input_names = op_.Input(name); - PADDLE_ENFORCE_GT(input_names.size(), 0UL, "Inputs(%s) length is 0", name); + PADDLE_ENFORCE(!input_names.empty(), "Inputs(%s) length is 0", name); for (auto& input : input_names) { if (!block_.HasVar(input)) return false; } return true; } - bool HasOutputs(const std::string& name) const { + bool HasOutputs(const std::string& name) const override { const std::vector& output_names = op_.Output(name); - PADDLE_ENFORCE_GT(output_names.size(), 0UL, "Inputs(%s) length is 0", name); + PADDLE_ENFORCE(!output_names.empty(), "Inputs(%s) length is 0", name); for (auto& output : output_names) { if (!block_.HasVar(output)) return false; } return true; } - DDim GetInputDim(const std::string& name) const { + DDim GetInputDim(const std::string& name) const override { std::vector ddims = GetInputsDim(name); PADDLE_ENFORCE_EQ(ddims.size(), 1UL, "Inputs(%s) length is not 1", name); return ddims[0]; } - void SetInputDim(const std::string& name, const DDim& dim) { + void SetInputDim(const std::string& name, const DDim& dim) override { SetInputsDim(name, {dim}); } - DDim GetOutputDim(const std::string& name) const { + DDim GetOutputDim(const std::string& name) const override { std::vector ddims = GetOutputsDim(name); PADDLE_ENFORCE_EQ(ddims.size(), 1UL, "Outputs(%s) length is not 1", name); return ddims[0]; } - void SetOutputDim(const std::string& name, const DDim& dim) { + void SetOutputDim(const std::string& name, const DDim& dim) override { SetOutputsDim(name, {dim}); } - AttrReader Attrs() const { return AttrReader(op_.GetAttrMap()); } + AttrReader Attrs() const override { return AttrReader(op_.GetAttrMap()); } - const std::vector& Inputs(const std::string& name) const { + const std::vector& Inputs( + const std::string& name) const override { return op_.Input(name); } - const std::vector& Outputs(const std::string& name) const { + const std::vector& Outputs( + const std::string& name) const override { return op_.Output(name); } private: - DDim GetDim(const std::string& name) const { + DDim GetDim(const std::string& name) const override { return framework::make_ddim(block_.Var(name)->Shape()); } - void SetDim(const std::string& name, const DDim& dim) { + void SetDim(const std::string& name, const DDim& dim) override { block_.Var(name)->SetShape(framework::vectorize(dim)); } @@ -403,21 +405,21 @@ class RuntimeInferShapeContext : public InferShapeContextBase { RuntimeInferShapeContext(const OperatorBase& op, const Scope& scope) : op_(op), scope_(scope) {} - bool HasInput(const std::string& name) const { + bool HasInput(const std::string& name) const override { auto ipt = op_.Input(name); auto* var = ipt == kEmptyVarName ? nullptr : scope_.FindVar(ipt); return var != nullptr; } - bool HasOutput(const std::string& name) const { + bool HasOutput(const std::string& name) const override { auto ipt = op_.Output(name); auto* var = ipt == kEmptyVarName ? nullptr : scope_.FindVar(ipt); return var != nullptr; } - bool HasInputs(const std::string& name) const { + bool HasInputs(const std::string& name) const override { auto inputs = op_.Inputs(name); - if (inputs.size() == 0UL) { + if (inputs.empty()) { return false; } for (auto& input : inputs) { @@ -428,9 +430,9 @@ class RuntimeInferShapeContext : public InferShapeContextBase { return true; } - bool HasOutputs(const std::string& name) const { + bool HasOutputs(const std::string& name) const override { auto outputs = op_.Outputs(name); - if (outputs.size() == 0UL) { + if (outputs.empty()) { return false; } for (auto& output : outputs) { @@ -441,29 +443,31 @@ class RuntimeInferShapeContext : public InferShapeContextBase { return true; } - DDim GetInputDim(const std::string& name) const { + DDim GetInputDim(const std::string& name) const override { return GetDim(op_.Input(name)); } - void SetInputDim(const std::string& name, const DDim& dim) { + void SetInputDim(const std::string& name, const DDim& dim) override { SetDim(op_.Input(name), dim); } - DDim GetOutputDim(const std::string& name) const { + DDim GetOutputDim(const std::string& name) const override { return GetDim(op_.Output(name)); } - void SetOutputDim(const std::string& name, const DDim& dim) { + void SetOutputDim(const std::string& name, const DDim& dim) override { SetDim(op_.Output(name), dim); } - AttrReader Attrs() const { return AttrReader(op_.Attrs()); } + AttrReader Attrs() const override { return AttrReader(op_.Attrs()); } - const std::vector& Inputs(const std::string& name) const { + const std::vector& Inputs( + const std::string& name) const override { return op_.Inputs(name); } - const std::vector& Outputs(const std::string& name) const { + const std::vector& Outputs( + const std::string& name) const override { return op_.Outputs(name); } @@ -484,11 +488,11 @@ class RuntimeInferShapeContext : public InferShapeContextBase { return t; } - DDim GetDim(const std::string& name) const { + DDim GetDim(const std::string& name) const override { return GetTensor(name)->dims(); } - void SetDim(const std::string& name, const DDim& dim) { + void SetDim(const std::string& name, const DDim& dim) override { GetTensor(name)->Resize(dim); } From 324876bbbfb0dd84f2172f951a2a4880bee32df4 Mon Sep 17 00:00:00 2001 From: Abhinav Arora Date: Tue, 3 Oct 2017 17:26:02 -0700 Subject: [PATCH 127/170] Changing learning rate from type Input(float) to Input(tensor) (#4578) --- paddle/operators/sgd_op.cc | 3 +++ paddle/operators/sgd_op.h | 2 +- python/paddle/v2/framework/tests/test_sgd_op.py | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/paddle/operators/sgd_op.cc b/paddle/operators/sgd_op.cc index 8f9eae4186..1a4d3fb8c5 100644 --- a/paddle/operators/sgd_op.cc +++ b/paddle/operators/sgd_op.cc @@ -32,6 +32,9 @@ class SGDOp : public framework::OperatorWithKernel { PADDLE_ENFORCE(ctx->HasOutput("param_out"), "Output(param_out) of SGDOp should not be null."); + auto lr_dims = ctx->GetInputDim("learning_rate"); + PADDLE_ENFORCE_EQ(framework::product(lr_dims), 1, + "Learning rate should have 1 element"); auto param_dim = ctx->GetInputDim("param"); PADDLE_ENFORCE_EQ(param_dim, ctx->GetInputDim("grad"), "Two input of SGD Op's dimension must be same."); diff --git a/paddle/operators/sgd_op.h b/paddle/operators/sgd_op.h index 977d201ced..e2ae65beb0 100644 --- a/paddle/operators/sgd_op.h +++ b/paddle/operators/sgd_op.h @@ -31,7 +31,7 @@ class SGDOpKernel : public framework::OpKernel { auto param = ctx.Input("param"); auto grad = ctx.Input("grad"); auto param_out = ctx.Output("param_out"); - float lr = *ctx.Input("learning_rate"); + float lr = ctx.Input("learning_rate")->data()[0]; param_out->mutable_data(ctx.GetPlace()); diff --git a/python/paddle/v2/framework/tests/test_sgd_op.py b/python/paddle/v2/framework/tests/test_sgd_op.py index f1125f4edb..c05364490f 100644 --- a/python/paddle/v2/framework/tests/test_sgd_op.py +++ b/python/paddle/v2/framework/tests/test_sgd_op.py @@ -8,7 +8,7 @@ class TestSGDOp(OpTest): self.op_type = "sgd" w = np.random.random((102, 105)).astype("float32") g = np.random.random((102, 105)).astype("float32") - lr = 0.1 + lr = np.array([0.1]).astype("float32") self.inputs = {'param': w, 'grad': g, 'learning_rate': lr} self.outputs = {'param_out': w - lr * g} From ab9545aa95fb482e7b51b58e0abe2191c9ef3bea Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Wed, 4 Oct 2017 00:44:07 -0700 Subject: [PATCH 128/170] add shape_inference_map --- paddle/framework/CMakeLists.txt | 4 +- paddle/framework/op_registry.h | 4 ++ paddle/framework/shape_inference.h | 1 + paddle/framework/shape_inference_map.cc | 57 +++++++++++++++++++++++++ paddle/framework/shape_inference_map.h | 56 ++++++++++++++++++++++++ 5 files changed, 121 insertions(+), 1 deletion(-) create mode 100644 paddle/framework/shape_inference_map.cc create mode 100644 paddle/framework/shape_inference_map.h diff --git a/paddle/framework/CMakeLists.txt b/paddle/framework/CMakeLists.txt index a2efcdb55c..986b45451f 100644 --- a/paddle/framework/CMakeLists.txt +++ b/paddle/framework/CMakeLists.txt @@ -26,8 +26,10 @@ cc_library(op_info SRCS op_info.cc DEPS attribute framework_proto proto_desc) cc_library(operator SRCS operator.cc DEPS op_info device_context tensor scope proto_desc) cc_test(operator_test SRCS operator_test.cc DEPS operator op_registry) +cc_library(shape_inference_map SRCS shape_inference_map.cc DEPS op_info operator) + cc_library(grad_op_builder SRCS grad_op_builder.cc DEPS operator proto_desc) -cc_library(op_registry SRCS op_registry.cc DEPS grad_op_builder op_proto_maker op_info) +cc_library(op_registry SRCS op_registry.cc DEPS grad_op_builder op_proto_maker op_info shape_inference_map) cc_test(op_registry_test SRCS op_registry_test.cc DEPS op_registry) cc_test(grad_op_builder_test SRCS grad_op_builder_test.cc DEPS grad_op_builder op_registry sum_op) diff --git a/paddle/framework/op_registry.h b/paddle/framework/op_registry.h index 4ee2c7d275..f04b6c503a 100644 --- a/paddle/framework/op_registry.h +++ b/paddle/framework/op_registry.h @@ -26,6 +26,7 @@ limitations under the License. */ #include "paddle/framework/grad_op_builder.h" #include "paddle/framework/operator.h" #include "paddle/framework/scope.h" +#include "paddle/framework/shape_inference_map.h" namespace paddle { namespace framework { @@ -54,9 +55,12 @@ class OpRegistry { const std::string& grad_op_type) { OperatorRegistrar reg(op_type.c_str()); reg.info.grad_op_type_ = grad_op_type; + ShapeInferenceMap::Instance().CreateOpWithKernel(reg.info, op_type); // register gradient op if (!grad_op_type.empty()) { OperatorRegistrar grad_reg(grad_op_type.c_str()); + ShapeInferenceMap::Instance().CreateOpWithKernel(grad_reg.info, + grad_op_type); } } diff --git a/paddle/framework/shape_inference.h b/paddle/framework/shape_inference.h index bc8af0eb3e..ac6f238638 100644 --- a/paddle/framework/shape_inference.h +++ b/paddle/framework/shape_inference.h @@ -14,6 +14,7 @@ limitations under the License. */ #pragma once +#include "paddle/framework/attribute.h" #include "paddle/framework/ddim.h" namespace paddle { diff --git a/paddle/framework/shape_inference_map.cc b/paddle/framework/shape_inference_map.cc new file mode 100644 index 0000000000..1a27037221 --- /dev/null +++ b/paddle/framework/shape_inference_map.cc @@ -0,0 +1,57 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/framework/shape_inference_map.h" + +namespace paddle { +namespace framework { + +static VariableNameMap ConvertOpProtoVarsToVarNameMap( + const google::protobuf::RepeatedPtrField& op_proto_vars) { + VariableNameMap ret_val; + for (auto& var : op_proto_vars) { + ret_val[var.name()] = {}; + } + return ret_val; +} + +static ShapeInferenceMap* g_shape_inference_map = nullptr; + +ShapeInferenceMap& ShapeInferenceMap::Instance() { + if (g_shape_inference_map == nullptr) { + g_shape_inference_map = new ShapeInferenceMap(); + } + return *g_shape_inference_map; +} + +void ShapeInferenceMap::CreateOpWithKernel(const OpInfo& op_info, + const std::string& op_type) { + const VariableNameMap inputs = + ConvertOpProtoVarsToVarNameMap(op_info.Proto().inputs()); + const VariableNameMap outputs = + ConvertOpProtoVarsToVarNameMap(op_info.Proto().outputs()); + auto* op = op_info.Creator()(op_type, inputs, outputs, {}); + auto* op_with_kernel = dynamic_cast(op); + auto it = op_shape_inference_map_.find(op_type); + if (it != op_shape_inference_map_.end()) { + PADDLE_THROW("OpWithKernel(%s) is already registered for infer_shape", + op_type); + } + if (op_with_kernel != nullptr) { + op_shape_inference_map_[op_type] = op_with_kernel; + } +} + +} // namespace framework +} // namespace paddle diff --git a/paddle/framework/shape_inference_map.h b/paddle/framework/shape_inference_map.h new file mode 100644 index 0000000000..fb12669026 --- /dev/null +++ b/paddle/framework/shape_inference_map.h @@ -0,0 +1,56 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include + +#include "paddle/framework/op_info.h" +#include "paddle/framework/operator.h" +#include "paddle/framework/shape_inference.h" + +namespace paddle { +namespace framework { + +class ShapeInferenceMap { + public: + static ShapeInferenceMap& Instance(); + + const OperatorBase* GetOperator(const std::string& op_type) { + auto it = op_shape_inference_map_.find(op_type); + if (it == op_shape_inference_map_.end()) { + PADDLE_THROW("op with kernel for Op(%s) is not registered", op_type); + } + return it->second; + } + + void CreateOpWithKernel(const OpInfo& op_info, const std::string& op_type); + + OperatorWithKernel* GetOpWithKernel(const std::string& op_type) { + auto it = op_shape_inference_map_.find(op_type); + if (it == op_shape_inference_map_.end()) { + return nullptr; + } + return it->second; + } + + private: + ShapeInferenceMap() = default; + DISABLE_COPY_AND_ASSIGN(ShapeInferenceMap); + + std::unordered_map op_shape_inference_map_; +}; + +} // namespace framework +} // namespace paddle From eed2c1e1d6237f421c9b8c0bbd2fd51d53beddcf Mon Sep 17 00:00:00 2001 From: Abhinav Arora Date: Wed, 4 Oct 2017 09:29:13 -0700 Subject: [PATCH 129/170] Changing SGD inputs and outputs to conform to Operator naming convention (#4586) --- paddle/operators/sgd_op.cc | 32 +++++++++---------- paddle/operators/sgd_op.h | 8 ++--- .../paddle/v2/framework/tests/test_sgd_op.py | 4 +-- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/paddle/operators/sgd_op.cc b/paddle/operators/sgd_op.cc index 1a4d3fb8c5..31d491f130 100644 --- a/paddle/operators/sgd_op.cc +++ b/paddle/operators/sgd_op.cc @@ -23,22 +23,22 @@ class SGDOp : public framework::OperatorWithKernel { protected: void InferShape(framework::InferShapeContextBase *ctx) const override { - PADDLE_ENFORCE(ctx->HasInput("param"), - "Input(param) of SGDOp should not be null."); - PADDLE_ENFORCE(ctx->HasInput("grad"), - "Input(grad) of SGDOp should not be null."); - PADDLE_ENFORCE(ctx->HasInput("learning_rate"), - "Input(learning_rate) of SGDOp should not be null."); - PADDLE_ENFORCE(ctx->HasOutput("param_out"), - "Output(param_out) of SGDOp should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Param"), + "Input(Param) of SGDOp should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Grad"), + "Input(Grad) of SGDOp should not be null."); + PADDLE_ENFORCE(ctx->HasInput("LearningRate"), + "Input(LearningRate) of SGDOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("ParamOut"), + "Output(ParamOut) of SGDOp should not be null."); - auto lr_dims = ctx->GetInputDim("learning_rate"); + auto lr_dims = ctx->GetInputDim("LearningRate"); PADDLE_ENFORCE_EQ(framework::product(lr_dims), 1, "Learning rate should have 1 element"); - auto param_dim = ctx->GetInputDim("param"); - PADDLE_ENFORCE_EQ(param_dim, ctx->GetInputDim("grad"), + auto param_dim = ctx->GetInputDim("Param"); + PADDLE_ENFORCE_EQ(param_dim, ctx->GetInputDim("Grad"), "Two input of SGD Op's dimension must be same."); - ctx->SetOutputDim("param_out", param_dim); + ctx->SetOutputDim("ParamOut", param_dim); } }; @@ -46,10 +46,10 @@ class SGDOpMaker : public framework::OpProtoAndCheckerMaker { public: SGDOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("param", "input parameter"); - AddInput("learning_rate", "learning rate of sgd"); - AddInput("grad", "input gradient"); - AddOutput("param_out", "output parameter"); + AddInput("Param", "Input parameter"); + AddInput("LearningRate", "Learning rate of SGD"); + AddInput("Grad", "Input gradient"); + AddOutput("ParamOut", "output parameter"); AddComment(R"DOC( Simplest sgd algorithm. diff --git a/paddle/operators/sgd_op.h b/paddle/operators/sgd_op.h index e2ae65beb0..d72d333a9a 100644 --- a/paddle/operators/sgd_op.h +++ b/paddle/operators/sgd_op.h @@ -28,10 +28,10 @@ template class SGDOpKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { - auto param = ctx.Input("param"); - auto grad = ctx.Input("grad"); - auto param_out = ctx.Output("param_out"); - float lr = ctx.Input("learning_rate")->data()[0]; + auto param = ctx.Input("Param"); + auto grad = ctx.Input("Grad"); + auto param_out = ctx.Output("ParamOut"); + float lr = ctx.Input("LearningRate")->data()[0]; param_out->mutable_data(ctx.GetPlace()); diff --git a/python/paddle/v2/framework/tests/test_sgd_op.py b/python/paddle/v2/framework/tests/test_sgd_op.py index c05364490f..2dd881e5e1 100644 --- a/python/paddle/v2/framework/tests/test_sgd_op.py +++ b/python/paddle/v2/framework/tests/test_sgd_op.py @@ -10,8 +10,8 @@ class TestSGDOp(OpTest): g = np.random.random((102, 105)).astype("float32") lr = np.array([0.1]).astype("float32") - self.inputs = {'param': w, 'grad': g, 'learning_rate': lr} - self.outputs = {'param_out': w - lr * g} + self.inputs = {'Param': w, 'Grad': g, 'LearningRate': lr} + self.outputs = {'ParamOut': w - lr * g} def test_check_output(self): self.check_output() From 5f51d0afc49f4bd4c624ec62aa1e3ccd31840aee Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Wed, 4 Oct 2017 10:00:39 -0700 Subject: [PATCH 130/170] Add -D PADDLE_WITH_CUDA in cmake/configure.cmake --- cmake/configure.cmake | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cmake/configure.cmake b/cmake/configure.cmake index 51c3b918cc..4e044ca421 100644 --- a/cmake/configure.cmake +++ b/cmake/configure.cmake @@ -49,11 +49,16 @@ if(NOT WITH_GOLANG) endif(NOT WITH_GOLANG) if(NOT WITH_GPU) + # Will gradually remove uses of PADDLE_ONLY_CPU in source files, + # so could we remove -DPADDLE_ONLY_CPU. + # c.f. https://github.com/PaddlePaddle/Paddle/issues/4588 add_definitions(-DPADDLE_ONLY_CPU) add_definitions(-DHPPL_STUB_FUNC) list(APPEND CMAKE_CXX_SOURCE_FILE_EXTENSIONS cu) else() + add_definitions(-DPADDLE_WITH_CUDA) + FIND_PACKAGE(CUDA REQUIRED) if(${CUDA_VERSION_MAJOR} VERSION_LESS 7) From 37b0bb15973632fca96fef31c8a5b30a78a80042 Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Wed, 4 Oct 2017 11:25:17 -0700 Subject: [PATCH 131/170] Fix compile errors --- paddle/framework/backward.cc | 54 ++++++++++++++++++----------------- paddle/framework/block_desc.h | 5 ++-- 2 files changed, 30 insertions(+), 29 deletions(-) diff --git a/paddle/framework/backward.cc b/paddle/framework/backward.cc index 4c76326e7c..a84262e007 100644 --- a/paddle/framework/backward.cc +++ b/paddle/framework/backward.cc @@ -14,9 +14,11 @@ #include "paddle/framework/backward.h" +#include #include #include +#include "paddle/framework/block_desc.h" #include "paddle/framework/op_registry.h" #include "paddle/operators/net_op.h" #include "paddle/operators/recurrent_op.h" @@ -254,23 +256,22 @@ static bool AllGradInSet(const std::vector& names, std::vector> MakeGradOpDescs( const std::unique_ptr& op_desc, - unordered_set& no_grad_vars) { + std::unordered_set& no_grad_vars) { std::vector> grad_op_descs; // All input gradients of forwarding operator do not need to calculat. - if (AllGradInSet(op_desc->InputArgumentNames(), kGradVarSuffix, - no_grad_vars)) { + if (AllGradInSet(op_desc->InputArgumentNames(), no_grad_vars)) { return grad_op_descs; // empty vector } // All output gradients of forwarding operator do not need to calculate. - const std::vector& outputs = op_desc->OutputArugumentNames(); - if (AllGradInSet(outputs, kGradVarSuffix, no_grad_vars)) { + const std::vector& outputs = op_desc->OutputArgumentNames(); + if (AllGradInSet(outputs, no_grad_vars)) { for (const std::string& name : outputs) { no_grad_vars.insert(GradVarName(name)); } return grad_op_descs; // empty vector } - grad_op_descs = OpRegistry::CreateGradOpDescs(op_desc); + grad_op_descs = OpRegistry::CreateGradOpDescs(*op_desc); std::list> pending_fill_zeros_ops; for (auto& desc : grad_op_descs) { @@ -280,43 +281,43 @@ std::vector> MakeGradOpDescs( 0, in_name.size() - sizeof(kGradVarSuffix) / sizeof(char) + 1); std::string new_name = prefix + kZeroVarSuffix; desc->Rename(in_name, new_name); - OpDescBind* fill_zeros_op = new OpDescBind( - "fill_zeros_like", {{"X", {prefix}}}, {{"Y", {new_name}}}, {}); - pending_fill_zeros_ops.push_back({fill_zeros_op}); + std::unique_ptr fill_zeros_op(new OpDescBind( + "fill_zeros_like", {{"X", {prefix}}}, {{"Y", {new_name}}}, {})); + pending_fill_zeros_ops.push_back(std::move(fill_zeros_op)); } } - for (const std::string& out_name : desc->OutputArgumentName()) { + for (const std::string& out_name : desc->OutputArgumentNames()) { if (no_grad_vars.count(out_name)) { desc->Rename(out_name, kEmptyVarName); } } } - grad_op_descs.insert(std::begin(grad_op_descs), - std::begin(pending_fill_zeros_ops), - std::end(pending_fill_zeros_ops)); + for (auto& p : pending_fill_zeros_ops) { + grad_op_descs.push_back(std::move(p)); + } - // TODO (fengjiayi): RNN op + // TODO(fengjiayi): RNN op return grad_op_descs; } -void AppendBackwardOpDescs( - BlockDescBind& block_desc, - const std::unordered_set& no_grad_vars) { +void AppendBackwardOpDescs(BlockDescBind& block_desc, + std::unordered_set& no_grad_vars) { std::unordered_map> dup_out_ops; size_t grad_desc_idx = 0; - std::deque> block_op_descs = block_desc.ops_; + std::deque>& block_op_descs = block_desc.ops_; std::vector> backward_descs; for (auto it = block_op_descs.rbegin(); it != block_op_descs.rend(); ++it) { std::vector> op_grads = MakeGradOpDescs(*it, no_grad_vars); for (const auto& desc : op_grads) { - for (const std::string& out_name : desc->OutputArugumentNames()) { + for (const std::string& out_name : desc->OutputArgumentNames()) { dup_out_ops[out_name].emplace_back(grad_desc_idx); } ++grad_desc_idx; } - backward_descs.insert(backward_descs.end(), op_grads.begin(), - op_grads.end()); + std::transform( + op_grads.begin(), op_grads.end(), std::back_inserter(backward_descs), + [](std::unique_ptr& ptr) { return std::move(ptr); }); } // Check whether some variables are written more than once std::list>> pending_sum_ops; @@ -330,9 +331,9 @@ void AppendBackwardOpDescs( backward_descs[dup_op[i]]->Rename(out_name, new_name); sum_op_inputs.emplace_back(new_name); } - OpDescBind* sum_op = new OpDescBind("sum", {{"X", sum_op_inputs}}, - {{"Out", {out_name}}}, {}); - pending_sum_ops.push_back({dup_op.back(), {sum_op}}); + std::unique_ptr sum_op(new OpDescBind( + "sum", {{"X", sum_op_inputs}}, {{"Out", {out_name}}}, {})); + pending_sum_ops.push_back({dup_op.back(), std::move(sum_op)}); } } pending_sum_ops.sort( @@ -345,8 +346,9 @@ void AppendBackwardOpDescs( std::move(p.second)); } // Append backward_descs to BlockDescBind::ops_ - block_op_descs.insert(std::end(block_op_descs), std::begin(backward_descs), - std::end(backward_descs)); + for (std::unique_ptr& ptr : backward_descs) { + block_op_descs.push_back(std::move(ptr)); + } return; } diff --git a/paddle/framework/block_desc.h b/paddle/framework/block_desc.h index a171dfef30..fd95ef1901 100644 --- a/paddle/framework/block_desc.h +++ b/paddle/framework/block_desc.h @@ -32,9 +32,8 @@ class ProgramDescBind; class BlockDescBind { public: - friend void AppendBackwardOps( - BlockDescBind &block_desc, - const std::unordered_set &no_grad_vars); + friend void AppendBackwardOpDescs( + BlockDescBind &block_desc, std::unordered_set &no_grad_vars); BlockDescBind(ProgramDescBind *prog, BlockDesc *desc) : prog_(prog), desc_(desc), need_update_(false) {} From 84500f9487164f3cf17625c876c15d754b932ced Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Wed, 4 Oct 2017 11:25:46 -0700 Subject: [PATCH 132/170] Change `PADDLE_ONLY_CPU` to `PADDLE_WITH_GPU` By shell command ```bash sed -i 's#ifdef PADDLE_ONLY_CPU#ifndef PADDLE_WITH_GPU#g' `find ./paddle/ -name '*.h' -o -name '*.cc' -o -name '*.cpp' -o -name '*.c' -o -name '*.cu'` sed -i 's#ifndef PADDLE_ONLY_CPU#ifdef PADDLE_WITH_GPU#g' `find ./paddle/ -name '*.h' -o -name '*.cc' -o -name '*.cpp' -o -name '*.c' -o -name '*.cu'` ``` --- cmake/configure.cmake | 2 +- paddle/api/Util.cpp | 2 +- paddle/capi/Matrix.cpp | 2 +- paddle/framework/lod_tensor.h | 4 +-- paddle/framework/op_registry.h | 2 +- paddle/framework/operator.cc | 2 +- paddle/framework/tensor_impl.h | 4 +-- paddle/framework/tensor_test.cc | 8 +++--- paddle/function/BlockExpandOp.cpp | 2 +- paddle/function/ContextProjectionOp.cpp | 2 +- paddle/function/CosSimOp.cpp | 2 +- paddle/function/CropOp.cpp | 2 +- paddle/function/CrossMapNormalOp.cpp | 2 +- paddle/function/DepthwiseConvOp.cpp | 2 +- paddle/function/DepthwiseConvOpTest.cpp | 2 +- paddle/function/GemmConvOp.cpp | 2 +- paddle/function/GemmConvOpTest.cpp | 2 +- paddle/function/Im2ColTest.cpp | 2 +- paddle/function/MulOp.cpp | 2 +- paddle/function/PadOp.cpp | 2 +- paddle/function/RowConvOp.cpp | 2 +- paddle/function/SwitchOp.cpp | 2 +- paddle/gserver/layers/BatchNormBaseLayer.cpp | 2 +- .../layers/BatchNormalizationLayer.cpp | 6 ++--- paddle/gserver/layers/PoolLayer.cpp | 4 +-- paddle/gserver/tests/LayerGradUtil.cpp | 2 +- paddle/gserver/tests/test_BatchNorm.cpp | 2 +- paddle/gserver/tests/test_ConvUnify.cpp | 2 +- paddle/gserver/tests/test_DetectionOutput.cpp | 2 +- paddle/gserver/tests/test_Evaluator.cpp | 2 +- paddle/gserver/tests/test_KmaxSeqScore.cpp | 2 +- paddle/gserver/tests/test_LayerGrad.cpp | 26 +++++++++---------- paddle/gserver/tests/test_NetworkCompare.cpp | 2 +- paddle/gserver/tests/test_PriorBox.cpp | 2 +- .../gserver/tests/test_ProtoDataProvider.cpp | 6 ++--- paddle/gserver/tests/test_PyDataProvider.cpp | 4 +-- .../gserver/tests/test_SelectiveFCLayer.cpp | 8 +++--- .../gserver/tests/test_SeqSliceLayerGrad.cpp | 2 +- paddle/gserver/tests/test_WarpCTCLayer.cpp | 2 +- paddle/math/Matrix.cpp | 6 ++--- paddle/math/SparseMatrix.cpp | 2 +- paddle/math/Vector.cpp | 6 ++--- paddle/math/tests/test_Allocator.cpp | 4 +-- paddle/math/tests/test_BaseMatrix.cpp | 2 +- paddle/math/tests/test_CpuGpuVector.cpp | 2 +- paddle/math/tests/test_ExecViaCpu.cpp | 2 +- paddle/math/tests/test_GpuProfiler.cpp | 2 +- paddle/math/tests/test_Matrix.cpp | 2 +- paddle/math/tests/test_SparseMatrix.cpp | 6 ++--- paddle/math/tests/test_Tensor.cu | 20 +++++++------- paddle/math/tests/test_TrainingAlgorithm.cpp | 2 +- paddle/math/tests/test_batchTranspose.cpp | 2 +- paddle/math/tests/test_lazyAssign.cu | 4 +-- paddle/math/tests/test_matrixCompare.cpp | 2 +- paddle/math/tests/test_perturbation.cpp | 2 +- .../math/tests/test_sparseMatrixCompare.cpp | 2 +- paddle/memory/detail/buddy_allocator.cc | 2 +- paddle/memory/detail/system_allocator.cc | 2 +- paddle/memory/detail/system_allocator.h | 2 +- paddle/memory/detail/system_allocator_test.cc | 2 +- paddle/memory/memcpy.cc | 2 +- paddle/memory/memcpy.h | 2 +- paddle/memory/memory.cc | 2 +- paddle/memory/memory_test.cc | 2 +- paddle/operators/detail/strided_memcpy.h | 2 +- paddle/operators/math/im2col_test.cc | 4 +-- paddle/operators/math/math_function_test.cc | 2 +- paddle/operators/strided_memcpy_test.cc | 2 +- paddle/platform/device_context.cc | 2 +- paddle/platform/device_context.h | 4 +-- paddle/platform/enforce.h | 4 +-- paddle/platform/gpu_info.h | 2 +- paddle/platform/variant.h | 2 +- paddle/pserver/test/SocketTest.cpp | 2 +- paddle/pserver/test/test_ProtoServer.cpp | 2 +- paddle/pybind/pybind.cc | 12 ++++----- paddle/pybind/tensor_py.h | 2 +- paddle/trainer/MergeModel.cpp | 2 +- paddle/trainer/tests/test_Compare.cpp | 2 +- paddle/trainer/tests/test_CompareSparse.cpp | 4 +-- paddle/trainer/tests/test_Trainer.cpp | 4 +-- paddle/trainer/tests/test_TrainerOnePass.cpp | 6 ++--- .../test_recurrent_machine_generation.cpp | 2 +- paddle/utils/Flags.cpp | 2 +- paddle/utils/Util.h | 2 +- paddle/utils/Version.h | 2 +- 86 files changed, 141 insertions(+), 141 deletions(-) diff --git a/cmake/configure.cmake b/cmake/configure.cmake index 51c3b918cc..926a7b1d69 100644 --- a/cmake/configure.cmake +++ b/cmake/configure.cmake @@ -49,11 +49,11 @@ if(NOT WITH_GOLANG) endif(NOT WITH_GOLANG) if(NOT WITH_GPU) - add_definitions(-DPADDLE_ONLY_CPU) add_definitions(-DHPPL_STUB_FUNC) list(APPEND CMAKE_CXX_SOURCE_FILE_EXTENSIONS cu) else() + add_definitions(-DPADDLE_WITH_GPU) FIND_PACKAGE(CUDA REQUIRED) if(${CUDA_VERSION_MAJOR} VERSION_LESS 7) diff --git a/paddle/api/Util.cpp b/paddle/api/Util.cpp index d369df5d4e..7446d892fd 100644 --- a/paddle/api/Util.cpp +++ b/paddle/api/Util.cpp @@ -47,7 +47,7 @@ bool isUsingGpu() { return FLAGS_use_gpu; } void setUseGpu(bool useGpu) { FLAGS_use_gpu = useGpu; } bool isGpuVersion() { -#ifdef PADDLE_ONLY_CPU +#ifndef PADDLE_WITH_GPU return false; #else return true; diff --git a/paddle/capi/Matrix.cpp b/paddle/capi/Matrix.cpp index d898ebe261..5b3737a759 100644 --- a/paddle/capi/Matrix.cpp +++ b/paddle/capi/Matrix.cpp @@ -46,7 +46,7 @@ paddle_error paddle_matrix_set_row(paddle_matrix mat, if (rowID >= ptr->mat->getHeight()) return kPD_OUT_OF_RANGE; paddle::real* buf = ptr->mat->getRowBuf(rowID); size_t width = ptr->mat->getWidth(); -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU hl_memcpy(buf, rowArray, sizeof(paddle::real) * width); #else std::copy(rowArray, rowArray + width, buf); diff --git a/paddle/framework/lod_tensor.h b/paddle/framework/lod_tensor.h index 49786a4a66..b12c95b6b7 100644 --- a/paddle/framework/lod_tensor.h +++ b/paddle/framework/lod_tensor.h @@ -15,7 +15,7 @@ #pragma once #include -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU #include #include #include @@ -29,7 +29,7 @@ namespace paddle { namespace framework { -#ifdef PADDLE_ONLY_CPU +#ifndef PADDLE_WITH_GPU template using Vector = std::vector; #else diff --git a/paddle/framework/op_registry.h b/paddle/framework/op_registry.h index 4ee2c7d275..aca6579f36 100644 --- a/paddle/framework/op_registry.h +++ b/paddle/framework/op_registry.h @@ -211,7 +211,7 @@ class OpKernelRegistrar : public Registrar { // TODO(fengjiayi): The following macros // seems ugly, do we have better method? -#ifdef PADDLE_ONLY_CPU +#ifndef PADDLE_WITH_GPU #define USE_OP_KERNEL(op_type) USE_OP_DEVICE_KERNEL(op_type, CPU) #else #define USE_OP_KERNEL(op_type) \ diff --git a/paddle/framework/operator.cc b/paddle/framework/operator.cc index 1012a30b0a..21c1c6f9e6 100644 --- a/paddle/framework/operator.cc +++ b/paddle/framework/operator.cc @@ -25,7 +25,7 @@ Eigen::DefaultDevice& ExecutionContext::GetEigenDevice< return *device_context_.GetEigenDevice(); } -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU template <> Eigen::GpuDevice& ExecutionContext::GetEigenDevice() const { diff --git a/paddle/framework/tensor_impl.h b/paddle/framework/tensor_impl.h index a5405f9c31..1cde1f74b8 100644 --- a/paddle/framework/tensor_impl.h +++ b/paddle/framework/tensor_impl.h @@ -65,7 +65,7 @@ inline T* Tensor::mutable_data(platform::Place place) { holder_.reset(new PlaceholderImpl( boost::get(place), size)); } else if (platform::is_gpu_place(place)) { -#ifdef PADDLE_ONLY_CPU +#ifndef PADDLE_WITH_GPU PADDLE_THROW("'GPUPlace' is not supported in CPU only device."); } #else @@ -103,7 +103,7 @@ inline void Tensor::CopyFrom(const Tensor& src, memory::Copy(boost::get(dst_place), dst_ptr, boost::get(src_place), src_ptr, size); } -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU else if (platform::is_gpu_place(src_place) && platform::is_cpu_place(dst_place)) { memory::Copy(boost::get(dst_place), dst_ptr, diff --git a/paddle/framework/tensor_test.cc b/paddle/framework/tensor_test.cc index e2ec738de3..86c6945ab5 100644 --- a/paddle/framework/tensor_test.cc +++ b/paddle/framework/tensor_test.cc @@ -74,7 +74,7 @@ TEST(Tensor, MutableData) { EXPECT_EQ(p1, p2); } -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU { Tensor src_tensor; float* p1 = nullptr; @@ -126,7 +126,7 @@ TEST(Tensor, ShareDataWith) { ASSERT_EQ(src_tensor.data(), dst_tensor.data()); } -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU { Tensor src_tensor; Tensor dst_tensor; @@ -163,7 +163,7 @@ TEST(Tensor, Slice) { EXPECT_EQ(src_data_address + 3 * 4 * 1 * sizeof(int), slice_data_address); } -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU { Tensor src_tensor; src_tensor.mutable_data(make_ddim({6, 9}), GPUPlace()); @@ -218,7 +218,7 @@ TEST(Tensor, CopyFrom) { EXPECT_EQ(dst_ptr[i], slice_ptr[i]); } } -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU { Tensor src_tensor; Tensor gpu_tensor; diff --git a/paddle/function/BlockExpandOp.cpp b/paddle/function/BlockExpandOp.cpp index a89b6bba45..ad78f5f584 100644 --- a/paddle/function/BlockExpandOp.cpp +++ b/paddle/function/BlockExpandOp.cpp @@ -194,7 +194,7 @@ public: REGISTER_TYPED_FUNC(BlockExpand, CPU, BlockExpandForward); REGISTER_TYPED_FUNC(BlockExpandGrad, CPU, BlockExpandBackward); -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU REGISTER_TYPED_FUNC(BlockExpand, GPU, BlockExpandForward); REGISTER_TYPED_FUNC(BlockExpandGrad, GPU, BlockExpandBackward); #endif diff --git a/paddle/function/ContextProjectionOp.cpp b/paddle/function/ContextProjectionOp.cpp index b87750b742..ab18c39df8 100644 --- a/paddle/function/ContextProjectionOp.cpp +++ b/paddle/function/ContextProjectionOp.cpp @@ -395,7 +395,7 @@ REGISTER_TYPED_FUNC(ContextProjectionForward, REGISTER_TYPED_FUNC(ContextProjectionBackward, CPU, ContextProjectionBackwardFunc); -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU REGISTER_TYPED_FUNC(ContextProjectionForward, GPU, ContextProjectionForwardFunc); diff --git a/paddle/function/CosSimOp.cpp b/paddle/function/CosSimOp.cpp index 7ece7b2dfe..4418f144d3 100644 --- a/paddle/function/CosSimOp.cpp +++ b/paddle/function/CosSimOp.cpp @@ -233,7 +233,7 @@ private: REGISTER_TYPED_FUNC(CosSimForward, CPU, CosSimForwardFunc); REGISTER_TYPED_FUNC(CosSimBackward, CPU, CosSimBackwardFunc); -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU REGISTER_TYPED_FUNC(CosSimForward, GPU, CosSimForwardFunc); REGISTER_TYPED_FUNC(CosSimBackward, GPU, CosSimBackwardFunc); #endif diff --git a/paddle/function/CropOp.cpp b/paddle/function/CropOp.cpp index f12ee43e3d..39504cc2c1 100644 --- a/paddle/function/CropOp.cpp +++ b/paddle/function/CropOp.cpp @@ -169,7 +169,7 @@ private: REGISTER_TYPED_FUNC(Crop, CPU, CropFunc); REGISTER_TYPED_FUNC(CropGrad, CPU, CropGradFunc); -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU REGISTER_TYPED_FUNC(Crop, GPU, CropFunc); REGISTER_TYPED_FUNC(CropGrad, GPU, CropGradFunc); #endif diff --git a/paddle/function/CrossMapNormalOp.cpp b/paddle/function/CrossMapNormalOp.cpp index ef878bfbba..1cf0918bed 100644 --- a/paddle/function/CrossMapNormalOp.cpp +++ b/paddle/function/CrossMapNormalOp.cpp @@ -336,7 +336,7 @@ private: REGISTER_TYPED_FUNC(CrossMapNormal, CPU, CrossMapNormalFunc); REGISTER_TYPED_FUNC(CrossMapNormalGrad, CPU, CrossMapNormalGradFunc); -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU REGISTER_TYPED_FUNC(CrossMapNormal, GPU, CrossMapNormalFunc); REGISTER_TYPED_FUNC(CrossMapNormalGrad, GPU, CrossMapNormalGradFunc); #endif diff --git a/paddle/function/DepthwiseConvOp.cpp b/paddle/function/DepthwiseConvOp.cpp index 2f3112fe65..7656ab3d0a 100644 --- a/paddle/function/DepthwiseConvOp.cpp +++ b/paddle/function/DepthwiseConvOp.cpp @@ -292,7 +292,7 @@ REGISTER_TYPED_FUNC(DepthwiseConvGradInput, REGISTER_TYPED_FUNC(DepthwiseConvGradFilter, CPU, DepthwiseConvGradFilterFunction); -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU REGISTER_TYPED_FUNC(DepthwiseConv, GPU, DepthwiseConvFunction); REGISTER_TYPED_FUNC(DepthwiseConvGradInput, GPU, diff --git a/paddle/function/DepthwiseConvOpTest.cpp b/paddle/function/DepthwiseConvOpTest.cpp index d8e8c889d5..39033ecb2b 100644 --- a/paddle/function/DepthwiseConvOpTest.cpp +++ b/paddle/function/DepthwiseConvOpTest.cpp @@ -17,7 +17,7 @@ limitations under the License. */ namespace paddle { -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU TEST(DepthwiseConv, Forward) { DepthwiseConvolution( "GemmConv-CPU", "DepthwiseConv-GPU", forward); diff --git a/paddle/function/GemmConvOp.cpp b/paddle/function/GemmConvOp.cpp index f8cf4ebea8..68e08c1480 100644 --- a/paddle/function/GemmConvOp.cpp +++ b/paddle/function/GemmConvOp.cpp @@ -340,7 +340,7 @@ public: REGISTER_TYPED_FUNC(GemmConv, CPU, GemmConvFunction); REGISTER_TYPED_FUNC(GemmConvGradInput, CPU, GemmConvGradInputFunction); REGISTER_TYPED_FUNC(GemmConvGradFilter, CPU, GemmConvGradFilterFunction); -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU REGISTER_TYPED_FUNC(GemmConv, GPU, GemmConvFunction); REGISTER_TYPED_FUNC(GemmConvGradInput, GPU, GemmConvGradInputFunction); REGISTER_TYPED_FUNC(GemmConvGradFilter, GPU, GemmConvGradFilterFunction); diff --git a/paddle/function/GemmConvOpTest.cpp b/paddle/function/GemmConvOpTest.cpp index 5283d79a5a..bd1cf3c6a4 100644 --- a/paddle/function/GemmConvOpTest.cpp +++ b/paddle/function/GemmConvOpTest.cpp @@ -24,7 +24,7 @@ TEST(GemmConv, NaiveConv) { "NaiveConv-CPU", "GemmConv-CPU", forward); } -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU TEST(GemmConv, Forward) { Convolution( "GemmConv-CPU", "GemmConv-GPU", forward); diff --git a/paddle/function/Im2ColTest.cpp b/paddle/function/Im2ColTest.cpp index acc88a553a..55325e94b5 100644 --- a/paddle/function/Im2ColTest.cpp +++ b/paddle/function/Im2ColTest.cpp @@ -116,7 +116,7 @@ void TestIm2ColFunctor() { TEST(Im2ColFunctor, CPU) { TestIm2ColFunctor(); } -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU TEST(Im2ColFunctor, GPU) { TestIm2ColFunctor(); } diff --git a/paddle/function/MulOp.cpp b/paddle/function/MulOp.cpp index 25e41edad5..655026320c 100644 --- a/paddle/function/MulOp.cpp +++ b/paddle/function/MulOp.cpp @@ -341,7 +341,7 @@ private: }; REGISTER_TYPED_FUNC(MulOp, CPU, MulFunc); -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU REGISTER_TYPED_FUNC(MulOp, GPU, MulFunc); #endif } // namespace paddle diff --git a/paddle/function/PadOp.cpp b/paddle/function/PadOp.cpp index adba7c92ec..24c9bf4e72 100644 --- a/paddle/function/PadOp.cpp +++ b/paddle/function/PadOp.cpp @@ -207,7 +207,7 @@ private: REGISTER_TYPED_FUNC(Pad, CPU, PadFunc); REGISTER_TYPED_FUNC(PadGrad, CPU, PadGradFunc); -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU REGISTER_TYPED_FUNC(Pad, GPU, PadFunc); REGISTER_TYPED_FUNC(PadGrad, GPU, PadGradFunc); #endif diff --git a/paddle/function/RowConvOp.cpp b/paddle/function/RowConvOp.cpp index b6501e8f4d..09e702f71a 100644 --- a/paddle/function/RowConvOp.cpp +++ b/paddle/function/RowConvOp.cpp @@ -217,7 +217,7 @@ public: REGISTER_TYPED_FUNC(RowConv, CPU, RowConvFunc); REGISTER_TYPED_FUNC(RowConvGrad, CPU, RowConvGradFunc); -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU REGISTER_TYPED_FUNC(RowConv, GPU, RowConvFunc); REGISTER_TYPED_FUNC(RowConvGrad, GPU, RowConvGradFunc); #endif diff --git a/paddle/function/SwitchOp.cpp b/paddle/function/SwitchOp.cpp index 01e252a8dc..db839b5b76 100644 --- a/paddle/function/SwitchOp.cpp +++ b/paddle/function/SwitchOp.cpp @@ -132,7 +132,7 @@ public: REGISTER_TYPED_FUNC(NCHW2NHWC, CPU, NCHW2NHWCFunc); REGISTER_TYPED_FUNC(NHWC2NCHW, CPU, NHWC2NCHWFunc); -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU REGISTER_TYPED_FUNC(NCHW2NHWC, GPU, NCHW2NHWCFunc); REGISTER_TYPED_FUNC(NHWC2NCHW, GPU, NHWC2NCHWFunc); #endif diff --git a/paddle/gserver/layers/BatchNormBaseLayer.cpp b/paddle/gserver/layers/BatchNormBaseLayer.cpp index f7a80e23e1..55f52816ab 100644 --- a/paddle/gserver/layers/BatchNormBaseLayer.cpp +++ b/paddle/gserver/layers/BatchNormBaseLayer.cpp @@ -16,7 +16,7 @@ limitations under the License. */ #include "BatchNormalizationLayer.h" #include "Layer.h" #include "paddle/utils/Stat.h" -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU #include "CudnnBatchNormLayer.h" #endif diff --git a/paddle/gserver/layers/BatchNormalizationLayer.cpp b/paddle/gserver/layers/BatchNormalizationLayer.cpp index 412762d384..33cf24431d 100644 --- a/paddle/gserver/layers/BatchNormalizationLayer.cpp +++ b/paddle/gserver/layers/BatchNormalizationLayer.cpp @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. */ #include "paddle/utils/Stat.h" -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU #include "hl_batch_transpose.h" #endif #include "BatchNormalizationLayer.h" @@ -90,7 +90,7 @@ void BatchNormalizationLayer::expandMat(const MatrixPtr& in, MatrixPtr& out) { size_t batchSize = in->getHeight(); CHECK_EQ(out->getHeight(), batchSize * imgPixels_); if (useGpu_) { -#ifdef PADDLE_ONLY_CPU +#ifndef PADDLE_WITH_GPU LOG(FATAL) << "paddle is compiled only for cpu"; #else batchTranspose( @@ -127,7 +127,7 @@ void BatchNormalizationLayer::shrinkMat(const MatrixPtr& in, MatrixPtr& out) { } CHECK_EQ(in->getHeight(), static_cast(batchSize * imgPixels_)); if (useGpu_) { -#ifdef PADDLE_ONLY_CPU +#ifndef PADDLE_WITH_GPU LOG(FATAL) << "paddle is compiled only for cpu"; #else batchTranspose( diff --git a/paddle/gserver/layers/PoolLayer.cpp b/paddle/gserver/layers/PoolLayer.cpp index 96d5c54acc..43ab4e4d47 100644 --- a/paddle/gserver/layers/PoolLayer.cpp +++ b/paddle/gserver/layers/PoolLayer.cpp @@ -15,7 +15,7 @@ limitations under the License. */ #include "PoolLayer.h" #include "PoolProjectionLayer.h" #include "paddle/utils/Logging.h" -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU #include "CudnnPoolLayer.h" #endif namespace paddle { @@ -53,7 +53,7 @@ Layer* PoolLayer::create(const LayerConfig& config) { const std::string& pool = config.inputs(0).pool_conf().pool_type(); if (pool == "max-projection" || pool == "avg-projection") { return new PoolProjectionLayer(config); -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU } else if (CudnnPoolLayer::typeCheck(pool)) { return new CudnnPoolLayer(config); #endif diff --git a/paddle/gserver/tests/LayerGradUtil.cpp b/paddle/gserver/tests/LayerGradUtil.cpp index a38880e14c..59df057a80 100644 --- a/paddle/gserver/tests/LayerGradUtil.cpp +++ b/paddle/gserver/tests/LayerGradUtil.cpp @@ -674,7 +674,7 @@ void testLayerGradKernel(TestConfig testConf, bool useGpu, bool useWeight, float epsilon) { -#ifdef PADDLE_ONLY_CPU +#ifndef PADDLE_WITH_GPU if (useGpu) return; #endif FLAGS_use_gpu = useGpu; diff --git a/paddle/gserver/tests/test_BatchNorm.cpp b/paddle/gserver/tests/test_BatchNorm.cpp index 659eefa31b..c1c85f8fac 100644 --- a/paddle/gserver/tests/test_BatchNorm.cpp +++ b/paddle/gserver/tests/test_BatchNorm.cpp @@ -119,7 +119,7 @@ TEST(Layer, batchNorm) { CHECK_EQ(static_cast(convLayer->getOutputValue()->getWidth()), 576); } -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU void batchNormInference(int n, int c, int h, int w) { MatrixPtr input = std::make_shared(n, c * h * w); MatrixPtr cudnnOut = std::make_shared(n, c * h * w); diff --git a/paddle/gserver/tests/test_ConvUnify.cpp b/paddle/gserver/tests/test_ConvUnify.cpp index e7325e0cc3..16556469cb 100644 --- a/paddle/gserver/tests/test_ConvUnify.cpp +++ b/paddle/gserver/tests/test_ConvUnify.cpp @@ -117,7 +117,7 @@ MatrixPtr doOneConvTest(size_t imgSize, } TEST(Layer, convParaUnified) { -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU MatrixPtr input, resultCpu, resultGpu; /// TEST1 for conv /// diff --git a/paddle/gserver/tests/test_DetectionOutput.cpp b/paddle/gserver/tests/test_DetectionOutput.cpp index af43dc51fa..1a83f48fae 100644 --- a/paddle/gserver/tests/test_DetectionOutput.cpp +++ b/paddle/gserver/tests/test_DetectionOutput.cpp @@ -150,7 +150,7 @@ TEST(Layer, detectionOutputLayerFwd) { useGpu, result2); -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU // GPU case 1. useGpu = true; inputLoc = Matrix::create(1, 16, false, useGpu); diff --git a/paddle/gserver/tests/test_Evaluator.cpp b/paddle/gserver/tests/test_Evaluator.cpp index 93996392d2..42bb570572 100644 --- a/paddle/gserver/tests/test_Evaluator.cpp +++ b/paddle/gserver/tests/test_Evaluator.cpp @@ -51,7 +51,7 @@ void testEvaluator(TestConfig testConf, string testEvaluatorName, size_t batchSize, bool useGpu) { -#ifdef PADDLE_ONLY_CPU +#ifndef PADDLE_WITH_GPU if (useGpu) return; #endif FLAGS_use_gpu = useGpu; diff --git a/paddle/gserver/tests/test_KmaxSeqScore.cpp b/paddle/gserver/tests/test_KmaxSeqScore.cpp index 308abe6816..1594de8502 100644 --- a/paddle/gserver/tests/test_KmaxSeqScore.cpp +++ b/paddle/gserver/tests/test_KmaxSeqScore.cpp @@ -97,7 +97,7 @@ TEST(Layer, kmaxSeqScoreLayer) { Matrix::create(subSeqStartPosition.back(), 1, false, false); std::vector mode = {false}; -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU mode.push_back(true); #endif diff --git a/paddle/gserver/tests/test_LayerGrad.cpp b/paddle/gserver/tests/test_LayerGrad.cpp index 090bde7b20..e887dee5f9 100644 --- a/paddle/gserver/tests/test_LayerGrad.cpp +++ b/paddle/gserver/tests/test_LayerGrad.cpp @@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU #include #endif #include @@ -258,7 +258,7 @@ void testProjectionConv(size_t groups, bool isDeconv) { true); } -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU TEST(Projection, conv) { /// test ConvProjection testProjectionConv(1, false); @@ -422,7 +422,7 @@ TEST(Layer, depthwiseConvLayer) { // 'depthwise_conv' is a sepecial case of 'exconv' whose // groups size equals to the input channels size. testDepthwiseConvLayer("exconv", /* useGpu= */ false); -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU testDepthwiseConvLayer("exconv", /* useGpu= */ true); #endif } @@ -480,7 +480,7 @@ void testConvLayer(const string& type, bool trans, bool useGpu) { TEST(Layer, convLayer) { testConvLayer("exconv", /* trans= */ false, /* useGpu= */ false); -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU testConvLayer("exconv", /* trans= */ false, /* useGpu= */ true); testConvLayer("cudnn_conv", /* trans= */ false, /* useGpu= */ true); #endif @@ -525,7 +525,7 @@ TEST(Layer, convTransLayer) { for (auto useGpu : {false, true}) { testConvTransLayer("exconvt", /* trans= */ false, /* useGpu= */ useGpu); } -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU testConvTransLayer("cudnn_convt", /* trans= */ false, /* useGpu= */ true); #endif } @@ -638,7 +638,7 @@ TEST(Layer, SelectiveFullyConnectedLayer) { /* trans= */ false, /* useGup= */ false, false); -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU testLayerGrad(config, "selective_fc", 100, @@ -1210,7 +1210,7 @@ void testPoolLayer(const string& poolType, bool trans, bool useGpu) { testLayerGrad(config, "pool", 100, trans, useGpu); } -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU void testPoolLayer2(const string& poolType, bool trans, bool useGpu) { TestConfig config; config.inputDefs.push_back({INPUT_DATA, "layer_0", 3200, 0}); @@ -1236,7 +1236,7 @@ TEST(Layer, PoolLayer) { testPoolLayer("avg-projection", /* trans= */ false, /* useGpu= */ false); testPoolLayer("max-projection", /* trans= */ false, /* useGpu= */ false); -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU testPoolLayer("avg-projection", /* trans= */ false, /* useGpu= */ true); testPoolLayer("max-projection", /* trans= */ false, /* useGpu= */ true); testPoolLayer("cudnn-max-pool", /* trans= */ false, /* useGpu= */ true); @@ -1309,7 +1309,7 @@ void testPool3DLayer(const string& poolType, bool trans, bool useGpu) { TEST(Layer, Pool3DLayer) { testPool3DLayer("avg", /* trans= */ false, /* useGpu= */ false); testPool3DLayer("max", /* trans= */ false, /* useGpu= */ false); -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU testPool3DLayer("avg", /* trans= */ false, /* useGpu= */ true); testPool3DLayer("max", /* trans= */ false, /* useGpu= */ true); #endif @@ -1695,7 +1695,7 @@ void testBatchNormLayer(const string& type, bool trans, bool useGpu) { TEST(Layer, BatchNormalizationLayer) { testBatchNormLayer("batch_norm", false, false); -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU testBatchNormLayer("batch_norm", false, true); if (hl_get_cudnn_lib_version() >= int(4000)) { testBatchNormLayer("cudnn_batch_norm", false, true); @@ -1744,7 +1744,7 @@ void testBatchNorm3DLayer(const string& type, bool trans, bool useGpu) { TEST(Layer, testBatchNorm3DLayer) { testBatchNorm3DLayer("batch_norm", false, false); -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU testBatchNorm3DLayer("batch_norm", false, true); if (hl_get_cudnn_lib_version() >= int(4000)) { testBatchNorm3DLayer("cudnn_batch_norm", false, true); @@ -2262,7 +2262,7 @@ void test3DConvLayer(const string& type, bool trans, bool useGpu) { TEST(Layer, test3DConvLayer) { test3DConvLayer("conv3d", /* trans= */ false, /* useGpu= */ false); -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU test3DConvLayer("conv3d", /* trans= */ false, /* useGpu= */ true); #endif } @@ -2339,7 +2339,7 @@ void test3DDeConvLayer(const string& type, bool trans, bool useGpu) { TEST(Layer, test3DDeConvLayer) { test3DDeConvLayer("deconv3d", /* trans= */ false, /* useGpu= */ false); -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU test3DDeConvLayer("deconv3d", /* trans= */ false, /* useGpu= */ true); #endif } diff --git a/paddle/gserver/tests/test_NetworkCompare.cpp b/paddle/gserver/tests/test_NetworkCompare.cpp index d36f72360f..e322fef9a4 100644 --- a/paddle/gserver/tests/test_NetworkCompare.cpp +++ b/paddle/gserver/tests/test_NetworkCompare.cpp @@ -243,7 +243,7 @@ TEST(Compare, concat_slice) { compareNetwork(config_file_a, config_file_b); } -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU TEST(Compare, img_pool) { std::string config_file_a = "./gserver/tests/img_pool_a.conf"; std::string config_file_b = "./gserver/tests/img_pool_b.conf"; diff --git a/paddle/gserver/tests/test_PriorBox.cpp b/paddle/gserver/tests/test_PriorBox.cpp index ae0e3bc3d2..cbc0fff7b8 100644 --- a/paddle/gserver/tests/test_PriorBox.cpp +++ b/paddle/gserver/tests/test_PriorBox.cpp @@ -151,7 +151,7 @@ TEST(Layer, priorBoxLayerFwd) { useGpu, result); -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU // reset the input parameters variance[1] = 0.1; variance[3] = 0.2; diff --git a/paddle/gserver/tests/test_ProtoDataProvider.cpp b/paddle/gserver/tests/test_ProtoDataProvider.cpp index e11bf402c2..988dbc2513 100644 --- a/paddle/gserver/tests/test_ProtoDataProvider.cpp +++ b/paddle/gserver/tests/test_ProtoDataProvider.cpp @@ -485,7 +485,7 @@ TEST(ProtoDataProvider, test) { // Currently in async mode, useGpu is not supported continue; } -#ifdef PADDLE_ONLY_CPU +#ifndef PADDLE_WITH_GPU if (useGpu) { continue; } @@ -525,7 +525,7 @@ TEST(ProtoDataProvider, constant_slots) { for (int numConstantSlots : {1, 2}) { for (int useGpu : numTwoArray) { for (int dataCompression : numTwoArray) { -#ifdef PADDLE_ONLY_CPU +#ifndef PADDLE_WITH_GPU if (useGpu) { continue; } @@ -708,7 +708,7 @@ TEST(ProtoSequenceDataProvider, test) { // Currently in async mode, useGpu is not supported continue; } -#ifdef PADDLE_ONLY_CPU +#ifndef PADDLE_WITH_GPU if (useGpu) { continue; } diff --git a/paddle/gserver/tests/test_PyDataProvider.cpp b/paddle/gserver/tests/test_PyDataProvider.cpp index db883543c3..f6522febf8 100644 --- a/paddle/gserver/tests/test_PyDataProvider.cpp +++ b/paddle/gserver/tests/test_PyDataProvider.cpp @@ -37,7 +37,7 @@ TEST(PyDataProvider, py_fill_slots) { config.clear_files(); std::string dataFile = "gserver/tests/pyDataProvider/pyDataProviderList"; config.set_files(dataFile); -#ifdef PADDLE_ONLY_CPU +#ifndef PADDLE_WITH_GPU bool useGpu = false; #else bool useGpu = true; @@ -71,7 +71,7 @@ TEST(PyDataProvider, py_fill_nest_slots) { std::string dataFile = "gserver/tests/pyDataProvider/pyDataProviderList"; config.set_files(dataFile); EXPECT_EQ(config.IsInitialized(), true); -#ifdef PADDLE_ONLY_CPU +#ifndef PADDLE_WITH_GPU bool useGpu = false; #else bool useGpu = true; diff --git a/paddle/gserver/tests/test_SelectiveFCLayer.cpp b/paddle/gserver/tests/test_SelectiveFCLayer.cpp index ab23d00a2c..b25d32fb2c 100644 --- a/paddle/gserver/tests/test_SelectiveFCLayer.cpp +++ b/paddle/gserver/tests/test_SelectiveFCLayer.cpp @@ -321,7 +321,7 @@ TEST(Layer, SelectiveFcLayer_train_dense_mul) { "filelist=gserver/tests/SelectiveFcTest/dense_mul_list"; for (auto useGpu : {false, true}) { -#ifdef PADDLE_ONLY_CPU +#ifndef PADDLE_WITH_GPU if (useGpu) { break; } @@ -388,7 +388,7 @@ void testSelectiveFcLayerTrainSparseMul(const LayerConfig& config, outMatSelfc->getWidth(), outMatSelfc->getElementCnt())); cpuOutMatSelfc->copyFrom(*outMatSelfc, HPPL_STREAM_DEFAULT); -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU if (useGpu) { hl_stream_synchronize(HPPL_STREAM_DEFAULT); } @@ -418,7 +418,7 @@ void testSelectiveFcLayerTrainSparseMul(const LayerConfig& config, MatrixPtr cpuOutMatFc( new CpuMatrix(outMatFc->getHeight(), outMatFc->getWidth())); cpuOutMatFc->copyFrom(*outMatFc, HPPL_STREAM_DEFAULT); -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU if (useGpu) { hl_stream_synchronize(HPPL_STREAM_DEFAULT); } @@ -443,7 +443,7 @@ TEST(Layer, SelectiveFcLayer_train_sparse_mul) { selLayerConfig.set_size(fcLayerWidth); testSelectiveFcLayerTrainSparseMul(selLayerConfig, false); -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU testSelectiveFcLayerTrainSparseMul(selLayerConfig, true); #endif } diff --git a/paddle/gserver/tests/test_SeqSliceLayerGrad.cpp b/paddle/gserver/tests/test_SeqSliceLayerGrad.cpp index e1d4ae1617..f28149081b 100644 --- a/paddle/gserver/tests/test_SeqSliceLayerGrad.cpp +++ b/paddle/gserver/tests/test_SeqSliceLayerGrad.cpp @@ -195,7 +195,7 @@ TEST(Layer, SeqSliceLayer) { vector> ends; std::vector mode = {false}; -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU mode.push_back(true); #endif genSeqInfo(seqStartPos, subSeqStartPos); diff --git a/paddle/gserver/tests/test_WarpCTCLayer.cpp b/paddle/gserver/tests/test_WarpCTCLayer.cpp index 55427e2f12..ae5b64257f 100644 --- a/paddle/gserver/tests/test_WarpCTCLayer.cpp +++ b/paddle/gserver/tests/test_WarpCTCLayer.cpp @@ -199,7 +199,7 @@ TEST(Layer, WarpCTCLayer) { for (auto batchSize : {1, 10, 32}) { for (auto normByTimes : {false, true}) { for (auto useGpu : {false, true}) { -#ifdef PADDLE_ONLY_CPU +#ifndef PADDLE_WITH_GPU if (useGpu) continue; #endif LOG(INFO) << "layerSize=" << layerSize << " batchSize=" << batchSize diff --git a/paddle/math/Matrix.cpp b/paddle/math/Matrix.cpp index 0023b4d0f5..de02f9c0d5 100644 --- a/paddle/math/Matrix.cpp +++ b/paddle/math/Matrix.cpp @@ -670,7 +670,7 @@ void GpuMatrix::leftMul(Matrix& a, real scaleAB, real scaleT) { } void GpuMatrix::selectRows(Matrix& table, IVector& ids) { -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU CHECK(dynamic_cast(&table)); CHECK(table.useGpu()); CHECK(ids.useGpu()); @@ -694,7 +694,7 @@ void GpuMatrix::selectRows(Matrix& table, IVector& ids) { } void GpuMatrix::addToRows(Matrix& table, IVector& ids) { -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU CHECK(dynamic_cast(&table)); CHECK(table.useGpu()); CHECK(ids.useGpu()); @@ -741,7 +741,7 @@ void GpuMatrix::rowMax(Matrix& max) { } void GpuMatrix::rowMax(IVector& maxIds, Matrix& maxVal) { -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU CHECK(maxIds.useGpu() && maxVal.useGpu()) << "Matrix type are not equal"; size_t numSamples = getHeight(); size_t beam = maxVal.getWidth(); diff --git a/paddle/math/SparseMatrix.cpp b/paddle/math/SparseMatrix.cpp index 6370c77386..1f31082ae8 100644 --- a/paddle/math/SparseMatrix.cpp +++ b/paddle/math/SparseMatrix.cpp @@ -836,7 +836,7 @@ void GpuSparseMatrix::zeroMem() { } void GpuSparseMatrix::rowMax(IVector& maxIds, Matrix& maxVal) { -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU CHECK(maxIds.useGpu() && maxVal.useGpu()) << "Matrix type are not equal"; size_t numSamples = getHeight(); size_t beam = maxVal.getWidth(); diff --git a/paddle/math/Vector.cpp b/paddle/math/Vector.cpp index eb87ee9bb7..54e57b255d 100644 --- a/paddle/math/Vector.cpp +++ b/paddle/math/Vector.cpp @@ -172,7 +172,7 @@ void GpuVectorT::isEqualTo(const VectorT& b, const T& value) { template void GpuVectorT::selectFrom(const VectorT& src, const VectorT& ids) { -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU hl_vector_select_from(this->getData(), this->getSize(), src.getData(), @@ -850,7 +850,7 @@ CpuGpuVectorT::CpuGpuVectorT(CpuGpuVectorT& src, size_t size) : sync_(nullptr) { CHECK_LE(offset + size, static_cast(src.getSize())); -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU SyncedFlag* flag = src.getSync(); if (*flag == DATA_AT_CPU) { src.copyToGpu(); // will set synchronous data between CPU and GPU @@ -861,7 +861,7 @@ CpuGpuVectorT::CpuGpuVectorT(CpuGpuVectorT& src, auto cMemHandle = (src.getVector(false))->getMemoryHandle(); cpuVectorT_ = std::make_shared>( size, std::dynamic_pointer_cast(cMemHandle), offset); -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU auto gMemHandle = (src.getVector(true))->getMemoryHandle(); gpuVectorT_ = std::make_shared>( size, std::dynamic_pointer_cast(gMemHandle), offset); diff --git a/paddle/math/tests/test_Allocator.cpp b/paddle/math/tests/test_Allocator.cpp index 1ca70ea84c..cf2f66aea1 100644 --- a/paddle/math/tests/test_Allocator.cpp +++ b/paddle/math/tests/test_Allocator.cpp @@ -68,7 +68,7 @@ void testPoolAllocator() { TEST(Allocator, Pool) { testPoolAllocator(); -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU testPoolAllocator(); #endif } @@ -92,7 +92,7 @@ TEST(MemoryHandle, Cpu) { EXPECT_EQ(ptr1, ptr2); } -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU TEST(MemoryHandle, Gpu) { int numGpu = hl_get_device_count(); diff --git a/paddle/math/tests/test_BaseMatrix.cpp b/paddle/math/tests/test_BaseMatrix.cpp index 22ce39701f..730759f3db 100644 --- a/paddle/math/tests/test_BaseMatrix.cpp +++ b/paddle/math/tests/test_BaseMatrix.cpp @@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU /** * This test file use autotest::AutoCompare and cmpWithoutArg to compares the * implementation of CPU and GPU member function in diff --git a/paddle/math/tests/test_CpuGpuVector.cpp b/paddle/math/tests/test_CpuGpuVector.cpp index 58bc43a38b..ccb4a902b0 100644 --- a/paddle/math/tests/test_CpuGpuVector.cpp +++ b/paddle/math/tests/test_CpuGpuVector.cpp @@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU #include #include "paddle/math/Vector.h" diff --git a/paddle/math/tests/test_ExecViaCpu.cpp b/paddle/math/tests/test_ExecViaCpu.cpp index 04c856453d..2d439cd060 100644 --- a/paddle/math/tests/test_ExecViaCpu.cpp +++ b/paddle/math/tests/test_ExecViaCpu.cpp @@ -94,7 +94,7 @@ void testWrapper(F&& f) { } } -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU TEST(ExecViaCpu, test1) { testWrapper(f); testWrapper(&f); diff --git a/paddle/math/tests/test_GpuProfiler.cpp b/paddle/math/tests/test_GpuProfiler.cpp index e6b5dba446..6dab187e3e 100644 --- a/paddle/math/tests/test_GpuProfiler.cpp +++ b/paddle/math/tests/test_GpuProfiler.cpp @@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU #include #include "paddle/math/Matrix.h" diff --git a/paddle/math/tests/test_Matrix.cpp b/paddle/math/tests/test_Matrix.cpp index 1c21da5b76..7a145eae6a 100644 --- a/paddle/math/tests/test_Matrix.cpp +++ b/paddle/math/tests/test_Matrix.cpp @@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU /** * This test file use autotest::AutoCompare and cmpWithArg to compares the * implementation of CPU and GPU member function in Matrix.cpp. diff --git a/paddle/math/tests/test_SparseMatrix.cpp b/paddle/math/tests/test_SparseMatrix.cpp index c0572dfdbf..8151dde106 100644 --- a/paddle/math/tests/test_SparseMatrix.cpp +++ b/paddle/math/tests/test_SparseMatrix.cpp @@ -47,7 +47,7 @@ struct MatrixPara { SparseFormat format; }; -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU void test_sparse_matrix_mul(MatrixPara paraA, MatrixPara paraB, MatrixPara paraC) { @@ -452,7 +452,7 @@ TEST(Matrix, SparseMatrixCSRFormatTrimFrom) { matB->trimFrom(*mat); checkSMatrixEqual2(matA, matB); -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU GpuSparseMatrixPtr matC = std::make_shared( height, trimedWidth, height, FLOAT_VALUE, SPARSE_CSR, true); matC->trimFrom(*mat); @@ -546,7 +546,7 @@ TEST(Matrix, SparseMatrixCSCFormatTrimFrom) { matB->trimFrom(*mat); checkSMatrixEqual2(matA, matB); -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU GpuSparseMatrixPtr matC = std::make_shared( height, trimedWidth, height, FLOAT_VALUE, SPARSE_CSC, true); matC->trimFrom(*mat); diff --git a/paddle/math/tests/test_Tensor.cu b/paddle/math/tests/test_Tensor.cu index 31b693afa8..d03698dee2 100644 --- a/paddle/math/tests/test_Tensor.cu +++ b/paddle/math/tests/test_Tensor.cu @@ -270,7 +270,7 @@ TEST(Unary, BaseOp) { TestUnaryVectorT testCpuIVector( testUnaryBaseOpInt); -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU TestUnaryMatrix testGpuMatrix(testUnaryBaseOp); TestUnaryVectorT testGpuVector(testUnaryBaseOp); TestUnaryVectorT testGpuIVector( @@ -317,7 +317,7 @@ void testUnayrMathOp(Tensor& A1, Tensor& A2) { TEST(Unary, MathOp) { TestUnaryMatrix testCpu(testUnayrMathOp); -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU TestUnaryMatrix testGpu(testUnayrMathOp); #endif } @@ -374,7 +374,7 @@ void testUnayrCompareOp(Tensor& A1, Tensor& A2) { TEST(Unary, CompareOp) { TestUnaryMatrix testCpu(testUnayrCompareOp); -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU TestUnaryMatrix testGpu(testUnayrCompareOp); #endif } @@ -536,7 +536,7 @@ void testBinaryBaseOp(Tensor& A1, Tensor& A2, Tensor& B) { TEST(Binary, BaseOp) { TestBinaryMatrix testCpu(testBinaryBaseOp); -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU TestBinaryMatrix testGpu(testBinaryBaseOp); #endif } @@ -710,7 +710,7 @@ void testBinaryMathOp(Tensor& A1, Tensor& A2, Tensor& B) { TEST(Binary, MathOp) { TestBinaryMatrix testCpu(testBinaryMathOp); -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU TestBinaryMatrix testGpu(testBinaryMathOp); #endif } @@ -810,7 +810,7 @@ void testBinaryCompareOp(Tensor& A1, Tensor& A2, Tensor& B) { TEST(Binary, CompareOp) { TestBinaryMatrix testCpu(testBinaryCompareOp); -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU TestBinaryMatrix testGpu(testBinaryCompareOp); #endif } @@ -955,7 +955,7 @@ void testTernaryBaseOp(Tensor& A1, Tensor& A2, Tensor& B, Tensor& C) { TEST(Ternary, BaseOp) { TestTernaryMatrix testCpu(testTernaryBaseOp); -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU TestTernaryMatrix testGpu(testTernaryBaseOp); #endif } @@ -1058,7 +1058,7 @@ void testTernaryCompareOp(Tensor& A1, Tensor& A2, Tensor& B, Tensor& C) { TEST(Ternary, CompareOp) { TestTernaryMatrix testCpu(testTernaryCompareOp); -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU TestTernaryMatrix testGpu(testTernaryCompareOp); #endif } @@ -1086,7 +1086,7 @@ void testQuaternaryAdd( TEST(Quaternary, BaseOp) { TestQuaternaryMatrix testCpu(testQuaternaryAdd); -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU TestQuaternaryMatrix testGpu(testQuaternaryAdd); #endif } @@ -1156,7 +1156,7 @@ void testQuaternaryCompareOp( TEST(Quaternary, CompareOp) { TestQuaternaryMatrix testCpu(testQuaternaryCompareOp); -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU TestQuaternaryMatrix testGpu(testQuaternaryCompareOp); #endif } diff --git a/paddle/math/tests/test_TrainingAlgorithm.cpp b/paddle/math/tests/test_TrainingAlgorithm.cpp index 4a88844b43..36ac024007 100644 --- a/paddle/math/tests/test_TrainingAlgorithm.cpp +++ b/paddle/math/tests/test_TrainingAlgorithm.cpp @@ -91,7 +91,7 @@ int VectorCheckErr(const VectorPtr& vector1, const VectorPtr& vector2) { typedef std::function testMatrixFunc; void testCase(testMatrixFunc matrixFunc) { -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU for (auto useGpu : {false, true}) { #else for (auto useGpu : {false}) { diff --git a/paddle/math/tests/test_batchTranspose.cpp b/paddle/math/tests/test_batchTranspose.cpp index 4eb9837909..0189e534eb 100644 --- a/paddle/math/tests/test_batchTranspose.cpp +++ b/paddle/math/tests/test_batchTranspose.cpp @@ -17,7 +17,7 @@ limitations under the License. */ using namespace paddle; // NOLINT -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU TEST(MatrixBatchTransTest, test_batch_matrix_transpose) { const int nx = 100; const int ny = 50; diff --git a/paddle/math/tests/test_lazyAssign.cu b/paddle/math/tests/test_lazyAssign.cu index 92afab4ff7..04f23cff55 100644 --- a/paddle/math/tests/test_lazyAssign.cu +++ b/paddle/math/tests/test_lazyAssign.cu @@ -72,7 +72,7 @@ void testLazyAssign(int height, int width) { TEST(lazyAssign, CPU) { testMatrixCase(testLazyAssign); } -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU TEST(lazyAssign, GPU) { testMatrixCase(testLazyAssign); } #endif @@ -142,6 +142,6 @@ void testSgdUpdate(int height, int width) { TEST(sgdUpdate, CPU) { testMatrixCase(testSgdUpdate); } -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU TEST(sgdUpdate, GPU) { testMatrixCase(testSgdUpdate); } #endif diff --git a/paddle/math/tests/test_matrixCompare.cpp b/paddle/math/tests/test_matrixCompare.cpp index 061fb22e3f..7735877ac8 100644 --- a/paddle/math/tests/test_matrixCompare.cpp +++ b/paddle/math/tests/test_matrixCompare.cpp @@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU /// This unittest checks GpuMatrix/CpuMatrix get same result, so disable when /// only cpu version. diff --git a/paddle/math/tests/test_perturbation.cpp b/paddle/math/tests/test_perturbation.cpp index 60ebae0153..dff18136ae 100644 --- a/paddle/math/tests/test_perturbation.cpp +++ b/paddle/math/tests/test_perturbation.cpp @@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU #include #include diff --git a/paddle/math/tests/test_sparseMatrixCompare.cpp b/paddle/math/tests/test_sparseMatrixCompare.cpp index a9185a4b24..e39cc0a2f6 100644 --- a/paddle/math/tests/test_sparseMatrixCompare.cpp +++ b/paddle/math/tests/test_sparseMatrixCompare.cpp @@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU /// This unittest checks GpuSparseMatrix/CpuSparseMatrix get same result, // so disable when /// only cpu version. diff --git a/paddle/memory/detail/buddy_allocator.cc b/paddle/memory/detail/buddy_allocator.cc index bb44970109..ed0c3374ff 100644 --- a/paddle/memory/detail/buddy_allocator.cc +++ b/paddle/memory/detail/buddy_allocator.cc @@ -175,7 +175,7 @@ void* BuddyAllocator::SystemAlloc(size_t size) { } BuddyAllocator::PoolSet::iterator BuddyAllocator::RefillPool() { -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU if (system_allocator_->UseGpu()) { if ((total_used_ + total_free_) == 0) { // Compute the maximum allocation size for the first allocation. diff --git a/paddle/memory/detail/system_allocator.cc b/paddle/memory/detail/system_allocator.cc index a270bd5958..64f8182b5c 100644 --- a/paddle/memory/detail/system_allocator.cc +++ b/paddle/memory/detail/system_allocator.cc @@ -62,7 +62,7 @@ void CPUAllocator::Free(void* p, size_t size, size_t index) { bool CPUAllocator::UseGpu() const { return false; } -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU void* GPUAllocator::Alloc(size_t& index, size_t size) { // CUDA documentation doesn't explain if cudaMalloc returns nullptr diff --git a/paddle/memory/detail/system_allocator.h b/paddle/memory/detail/system_allocator.h index 82ba322e05..6b1f40347b 100644 --- a/paddle/memory/detail/system_allocator.h +++ b/paddle/memory/detail/system_allocator.h @@ -40,7 +40,7 @@ class CPUAllocator : public SystemAllocator { virtual bool UseGpu() const; }; -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU class GPUAllocator : public SystemAllocator { public: virtual void* Alloc(size_t& index, size_t size); diff --git a/paddle/memory/detail/system_allocator_test.cc b/paddle/memory/detail/system_allocator_test.cc index ba44e06ddb..57d5443d50 100644 --- a/paddle/memory/detail/system_allocator_test.cc +++ b/paddle/memory/detail/system_allocator_test.cc @@ -56,7 +56,7 @@ TEST(CPUAllocator, LockMem) { TestAllocator(a, 0); } -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU TEST(GPUAllocator, Alloc) { paddle::memory::detail::GPUAllocator a; TestAllocator(a, 2048); diff --git a/paddle/memory/memcpy.cc b/paddle/memory/memcpy.cc index c96a697a7e..184d0f8fa7 100644 --- a/paddle/memory/memcpy.cc +++ b/paddle/memory/memcpy.cc @@ -26,7 +26,7 @@ void Copy(platform::CPUPlace, void* dst, std::memcpy(dst, src, num); } -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU template <> void Copy(platform::CPUPlace dst_place, void* dst, diff --git a/paddle/memory/memcpy.h b/paddle/memory/memcpy.h index 2b9c0eada6..7142831d43 100644 --- a/paddle/memory/memcpy.h +++ b/paddle/memory/memcpy.h @@ -33,7 +33,7 @@ namespace memory { template void Copy(DstPlace, void* dst, SrcPlace, const void* src, size_t num); -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU /** * \brief Copy memory from one place to another place. diff --git a/paddle/memory/memory.cc b/paddle/memory/memory.cc index 29bc26f9d3..6d5a74dafe 100644 --- a/paddle/memory/memory.cc +++ b/paddle/memory/memory.cc @@ -62,7 +62,7 @@ size_t Used(platform::CPUPlace place) { return GetCPUBuddyAllocator()->Used(); } -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU BuddyAllocator* GetGPUBuddyAllocator(int gpu_id) { using BuddyAllocVec = std::vector; diff --git a/paddle/memory/memory_test.cc b/paddle/memory/memory_test.cc index 53cc63a098..7a617f04dc 100644 --- a/paddle/memory/memory_test.cc +++ b/paddle/memory/memory_test.cc @@ -80,7 +80,7 @@ TEST(BuddyAllocator, CPUMultAlloc) { } } -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU size_t align(size_t size, paddle::platform::GPUPlace place) { size += sizeof(paddle::memory::detail::Metadata); diff --git a/paddle/operators/detail/strided_memcpy.h b/paddle/operators/detail/strided_memcpy.h index b165224b37..9f05a26322 100644 --- a/paddle/operators/detail/strided_memcpy.h +++ b/paddle/operators/detail/strided_memcpy.h @@ -34,7 +34,7 @@ struct StridedMemcpyFunctor { auto& cpu_place = boost::get(place); memory::Copy(cpu_place, dst, cpu_place, src, sizeof(T) * dst_dim.head); } else { -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU auto& gpu_place = boost::get(place); auto& cuda_ctx = reinterpret_cast(dev_ctx); diff --git a/paddle/operators/math/im2col_test.cc b/paddle/operators/math/im2col_test.cc index f0b8c88591..3d040ca2b5 100644 --- a/paddle/operators/math/im2col_test.cc +++ b/paddle/operators/math/im2col_test.cc @@ -71,7 +71,7 @@ void testIm2col() { context = new paddle::platform::CPUDeviceContext(paddle::platform::CPUPlace()); } else { -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU context = new paddle::platform::CUDADeviceContext(paddle::platform::GPUPlace()); #else @@ -116,7 +116,7 @@ void testIm2col() { TEST(math, im2col) { testIm2col(); -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU testIm2col(); #endif } diff --git a/paddle/operators/math/math_function_test.cc b/paddle/operators/math/math_function_test.cc index 22468a0c4a..2252268620 100644 --- a/paddle/operators/math/math_function_test.cc +++ b/paddle/operators/math/math_function_test.cc @@ -1,7 +1,7 @@ #include "paddle/operators/math/math_function.h" #include "gtest/gtest.h" -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU TEST(math_function, notrans_mul_trans) { paddle::framework::Tensor input1; paddle::framework::Tensor input1_gpu; diff --git a/paddle/operators/strided_memcpy_test.cc b/paddle/operators/strided_memcpy_test.cc index 05882a8873..e0dd7b19f1 100644 --- a/paddle/operators/strided_memcpy_test.cc +++ b/paddle/operators/strided_memcpy_test.cc @@ -72,7 +72,7 @@ TEST(StridedMemcpy, CPUConcat) { } } -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU TEST(StridedMemcpy, GPUCrop) { // clang-format off int src[] = { diff --git a/paddle/platform/device_context.cc b/paddle/platform/device_context.cc index 36af1ac677..8dcc357a16 100644 --- a/paddle/platform/device_context.cc +++ b/paddle/platform/device_context.cc @@ -35,7 +35,7 @@ Eigen::DefaultDevice* CPUDeviceContext::eigen_device() const { Place CPUDeviceContext::GetPlace() const { return CPUPlace(); } -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU template <> Eigen::GpuDevice* diff --git a/paddle/platform/device_context.h b/paddle/platform/device_context.h index d805d2ab08..c1c4c7f760 100644 --- a/paddle/platform/device_context.h +++ b/paddle/platform/device_context.h @@ -14,7 +14,7 @@ limitations under the License. */ #include "paddle/platform/enforce.h" #include "paddle/platform/place.h" -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU #include "paddle/platform/dynload/cublas.h" #include "paddle/platform/dynload/cudnn.h" #include "paddle/platform/gpu_info.h" @@ -61,7 +61,7 @@ class CPUDeviceContext : public DeviceContext { std::unique_ptr eigen_device_; }; -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU template <> struct EigenDeviceConverter { using EigenDeviceType = Eigen::GpuDevice; diff --git a/paddle/platform/enforce.h b/paddle/platform/enforce.h index 52bd23039b..f9fe521d50 100644 --- a/paddle/platform/enforce.h +++ b/paddle/platform/enforce.h @@ -29,7 +29,7 @@ limitations under the License. */ #include // for __cxa_demangle #endif -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU #include "paddle/platform/dynload/cublas.h" #include "paddle/platform/dynload/cudnn.h" @@ -113,7 +113,7 @@ inline typename std::enable_if::type throw_on_error( } } -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU template inline typename std::enable_if::type throw_on_error( diff --git a/paddle/platform/gpu_info.h b/paddle/platform/gpu_info.h index f0c825bd9b..ac884386dd 100644 --- a/paddle/platform/gpu_info.h +++ b/paddle/platform/gpu_info.h @@ -14,7 +14,7 @@ limitations under the License. */ #pragma once -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU #include #include diff --git a/paddle/platform/variant.h b/paddle/platform/variant.h index 16ee00efe7..8145799dfd 100644 --- a/paddle/platform/variant.h +++ b/paddle/platform/variant.h @@ -16,7 +16,7 @@ #include -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU // Because boost's variadic templates has bug on nvcc, boost will disable // variadic template support when GPU enabled on nvcc. diff --git a/paddle/pserver/test/SocketTest.cpp b/paddle/pserver/test/SocketTest.cpp index 6f6c9e596c..96724530f5 100644 --- a/paddle/pserver/test/SocketTest.cpp +++ b/paddle/pserver/test/SocketTest.cpp @@ -215,7 +215,7 @@ int main(int argc, char** argv) { uint64_t dataSize = FLAGS_dim * sizeof(real); -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU GpuVector gpuParam(FLAGS_dim); GpuVector gpuGrad(FLAGS_dim); #else diff --git a/paddle/pserver/test/test_ProtoServer.cpp b/paddle/pserver/test/test_ProtoServer.cpp index 04236fda2f..74ab1f2f77 100644 --- a/paddle/pserver/test/test_ProtoServer.cpp +++ b/paddle/pserver/test/test_ProtoServer.cpp @@ -99,7 +99,7 @@ TEST(ProtoServer, regular) { } TEST(ProtoServer, extended) { -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU ProtoClient* client; if (FLAGS_rdma_tcp == "rdma") client = new ProtoClient(FLAGS_server_addr, FLAGS_port, F_RDMA); diff --git a/paddle/pybind/pybind.cc b/paddle/pybind/pybind.cc index d480427f59..761d82fc4d 100644 --- a/paddle/pybind/pybind.cc +++ b/paddle/pybind/pybind.cc @@ -34,7 +34,7 @@ static size_t UniqueIntegerGenerator() { } bool IsCompileGPU() { -#ifdef PADDLE_ONLY_CPU +#ifndef PADDLE_WITH_GPU return false; #else return true; @@ -78,7 +78,7 @@ PYBIND11_PLUGIN(core) { .def("set", PyCPUTensorSetFromArray) .def("set", PyCPUTensorSetFromArray) .def("set", PyCPUTensorSetFromArray) -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU .def("set", PyCUDATensorSetFromArray) .def("set", PyCUDATensorSetFromArray) .def("set", PyCUDATensorSetFromArray) @@ -96,7 +96,7 @@ PYBIND11_PLUGIN(core) { .def( "__init__", [](LoDTensor &instance, const std::vector> &lod) { -#ifdef PADDLE_ONLY_CPU +#ifndef PADDLE_WITH_GPU new (&instance) LoDTensor(lod); #else LoD new_lod; @@ -107,7 +107,7 @@ PYBIND11_PLUGIN(core) { }) .def("set_lod", [](LoDTensor &self, const std::vector> &lod) { -#ifdef PADDLE_ONLY_CPU +#ifndef PADDLE_WITH_GPU self.set_lod(lod); #else LoD new_lod; @@ -117,7 +117,7 @@ PYBIND11_PLUGIN(core) { #endif }) .def("lod", [](LoDTensor &self) -> std::vector> { -#ifdef PADDLE_ONLY_CPU +#ifndef PADDLE_WITH_GPU return self.lod(); #else auto lod = self.lod(); @@ -203,7 +203,7 @@ All parameter, weight, gradient are variables in Paddle. .def_static("create", [](paddle::platform::GPUPlace& place) -> paddle::platform::DeviceContext* { -#ifdef PADDLE_ONLY_CPU +#ifndef PADDLE_WITH_GPU PADDLE_THROW("GPUPlace is not supported in CPU device."); #else return new paddle::platform::CUDADeviceContext(place); diff --git a/paddle/pybind/tensor_py.h b/paddle/pybind/tensor_py.h index 3e3e6bc031..62e85fa54f 100644 --- a/paddle/pybind/tensor_py.h +++ b/paddle/pybind/tensor_py.h @@ -106,7 +106,7 @@ void PyCPUTensorSetFromArray( std::memcpy(dst, array.data(), sizeof(T) * array.size()); } -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU template void PyCUDATensorSetFromArray( framework::Tensor &self, diff --git a/paddle/trainer/MergeModel.cpp b/paddle/trainer/MergeModel.cpp index 91d89b61a3..a37d53bc72 100644 --- a/paddle/trainer/MergeModel.cpp +++ b/paddle/trainer/MergeModel.cpp @@ -29,7 +29,7 @@ int main(int argc, char** argv) { initMain(argc, argv); initPython(argc, argv); string confFile = TrainerConfigHelper::getConfigNameFromPath(FLAGS_model_dir); -#ifdef PADDLE_ONLY_CPU +#ifndef PADDLE_WITH_GPU FLAGS_use_gpu = false; #endif auto config = std::make_shared(confFile); diff --git a/paddle/trainer/tests/test_Compare.cpp b/paddle/trainer/tests/test_Compare.cpp index e855a8fe2e..b5d29da45a 100644 --- a/paddle/trainer/tests/test_Compare.cpp +++ b/paddle/trainer/tests/test_Compare.cpp @@ -146,7 +146,7 @@ void compareGradient(comData& comDataCpu, comData& comDataGpu) { } int main(int argc, char** argv) { -#ifdef PADDLE_ONLY_CPU +#ifndef PADDLE_WITH_GPU exit(0); #endif paddle::initMain(argc, argv); diff --git a/paddle/trainer/tests/test_CompareSparse.cpp b/paddle/trainer/tests/test_CompareSparse.cpp index 813275518e..4da9ce20fb 100644 --- a/paddle/trainer/tests/test_CompareSparse.cpp +++ b/paddle/trainer/tests/test_CompareSparse.cpp @@ -174,7 +174,7 @@ TEST(compareSparse, multiGradientMachine) { FLAGS_local = local; FLAGS_ports_num_for_sparse = 5; for (bool useGpu : {false, true}) { -#ifdef PADDLE_ONLY_CPU +#ifndef PADDLE_WITH_GPU if (useGpu) continue; #endif FLAGS_parallel_nn = useGpu; @@ -198,7 +198,7 @@ TEST(compareSparse, NeuralNetwork) { FLAGS_local = local; FLAGS_ports_num_for_sparse = 5; for (bool useGpu : {false, true}) { -#ifdef PADDLE_ONLY_CPU +#ifndef PADDLE_WITH_GPU if (useGpu) continue; #endif FLAGS_parallel_nn = useGpu; diff --git a/paddle/trainer/tests/test_Trainer.cpp b/paddle/trainer/tests/test_Trainer.cpp index 264bc46ebc..f69e1aafee 100644 --- a/paddle/trainer/tests/test_Trainer.cpp +++ b/paddle/trainer/tests/test_Trainer.cpp @@ -51,7 +51,7 @@ void checkGradientTest(const string& configFile, TEST(checkGradient, cpu) { checkGradientTest(configFile1, false, false); } -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU TEST(checkGradient, gpu) { checkGradientTest(configFile1, true, false); } TEST(checkGradient, multiGpu) { @@ -97,7 +97,7 @@ TEST(checkGradient, hsigmoid) { checkGradientTest(configFile2, false, false); } TEST(checkGradient, chunk) { checkGradientTest(configFile3, false, false); -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU checkGradientTest(configFile3, true, true); #endif } diff --git a/paddle/trainer/tests/test_TrainerOnePass.cpp b/paddle/trainer/tests/test_TrainerOnePass.cpp index 00ba61377a..4c4d124fa9 100644 --- a/paddle/trainer/tests/test_TrainerOnePass.cpp +++ b/paddle/trainer/tests/test_TrainerOnePass.cpp @@ -79,7 +79,7 @@ void trainerOnePassTest(const string& configFile, // 1. test trainer (cpu, gpu). TEST(trainerOnePass, cpu) { trainerOnePassTest(configFile1, false, false); } -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU TEST(trainerOnePass, gpu) { trainerOnePassTest(configFile1, true, false); } TEST(trainerOnePass, gpu2) { trainerOnePassTest(configFile1, true, false, 2); } @@ -94,7 +94,7 @@ TEST(trainerOnePass, parallel) { #endif // 2. test average_window. -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU TEST(average_window, gpu) { trainerOnePassTest(configFile1, true, false, 4, 0.01); } @@ -266,7 +266,7 @@ TEST(checkRemoteUpdater, cpuTrainerOldUpdater) { checkRemoteParameterUpdaterTest(configFile1, false, false, 1, true); } -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU TEST(checkRemoteUpdater, gpuTrainer) { checkRemoteParameterUpdaterTest(configFile1, true, false); } diff --git a/paddle/trainer/tests/test_recurrent_machine_generation.cpp b/paddle/trainer/tests/test_recurrent_machine_generation.cpp index 1322e77178..74b4fed7ed 100644 --- a/paddle/trainer/tests/test_recurrent_machine_generation.cpp +++ b/paddle/trainer/tests/test_recurrent_machine_generation.cpp @@ -113,7 +113,7 @@ void testGeneration(const string& configFile, #ifndef PADDLE_TYPE_DOUBLE TEST(RecurrentGradientMachine, test_generation) { -#ifdef PADDLE_ONLY_CPU +#ifndef PADDLE_WITH_GPU const auto useGpuConfs = {false}; #else const auto useGpuConfs = {true, false}; diff --git a/paddle/utils/Flags.cpp b/paddle/utils/Flags.cpp index ab1c181c62..32155ded35 100644 --- a/paddle/utils/Flags.cpp +++ b/paddle/utils/Flags.cpp @@ -14,7 +14,7 @@ limitations under the License. */ #include "Flags.h" -#ifdef PADDLE_ONLY_CPU +#ifndef PADDLE_WITH_GPU DEFINE_bool(use_gpu, false, "Only support CPU training"); #else DEFINE_bool(use_gpu, true, "Whether to use GPU for training"); diff --git a/paddle/utils/Util.h b/paddle/utils/Util.h index 22ce2534d3..904d0f5061 100644 --- a/paddle/utils/Util.h +++ b/paddle/utils/Util.h @@ -218,7 +218,7 @@ protected: * *d2* is peer device to enable direct access to by the d1 device. */ inline void enablePeerAccess(int d1, int d2) { -#ifndef PADDLE_ONLY_CPU +#ifdef PADDLE_WITH_GPU if (hl_device_can_access_peer(d1, d2)) { SetDevice dev(d1); hl_device_enable_peer_access(d2); diff --git a/paddle/utils/Version.h b/paddle/utils/Version.h index f53d6420bb..611fda83d9 100644 --- a/paddle/utils/Version.h +++ b/paddle/utils/Version.h @@ -48,7 +48,7 @@ void printVersion(std::ostream& os); * @return return true if paddle compiled with GPU */ constexpr bool isWithGpu() { -#ifdef PADDLE_ONLY_CPU +#ifndef PADDLE_WITH_GPU return false; #else return true; From e119177a8c2372335573d27f273c286580db2bd8 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Wed, 4 Oct 2017 12:11:34 -0700 Subject: [PATCH 133/170] Use unique_ptr --- paddle/framework/backward.cc | 8 +++--- paddle/framework/backward_test.cc | 20 +++++++-------- paddle/operators/mean_op.cc | 14 +++++------ paddle/operators/minus_op.cc | 25 +++++++++++-------- paddle/operators/pad_op.cc | 20 +++++++-------- paddle/operators/scale_op.cc | 14 +++++------ .../softmax_with_cross_entropy_op.cc | 22 ++++++++-------- paddle/operators/sum_op.cc | 17 +++++++------ 8 files changed, 72 insertions(+), 68 deletions(-) diff --git a/paddle/framework/backward.cc b/paddle/framework/backward.cc index 22c8c83f13..40390d4150 100644 --- a/paddle/framework/backward.cc +++ b/paddle/framework/backward.cc @@ -36,9 +36,11 @@ static inline std::unique_ptr CreateGradOp( auto grad_descs = info.grad_op_maker_(op_desc); std::vector> grad_ops; grad_ops.reserve(grad_descs.size()); - std::transform( - grad_descs.begin(), grad_descs.end(), std::back_inserter(grad_ops), - [](OpDescBind& grad_desc) { return OpRegistry::CreateOp(&grad_desc); }); + std::transform(grad_descs.begin(), grad_descs.end(), + std::back_inserter(grad_ops), + [](const std::unique_ptr& grad_desc) { + return OpRegistry::CreateOp(grad_desc.get()); + }); PADDLE_ENFORCE_GT(grad_ops.size(), 0); if (grad_ops.size() == 1) { return std::move(grad_ops[0]); diff --git a/paddle/framework/backward_test.cc b/paddle/framework/backward_test.cc index c88e85f8c4..830d0427fa 100644 --- a/paddle/framework/backward_test.cc +++ b/paddle/framework/backward_test.cc @@ -39,13 +39,13 @@ class RowWiseAddGradMaker : public SingleGradOpDescMaker { using SingleGradOpDescMaker::SingleGradOpDescMaker; protected: - OpDescBind Apply() const override { - OpDescBind grad_op; - grad_op.SetInput(GradVarName("Out"), OutputGrad("Out")); - grad_op.SetOutput(GradVarName("X"), InputGrad("X")); - grad_op.SetOutput(GradVarName("b"), InputGrad("b")); - grad_op.SetType("rowwise_add_grad"); - return grad_op; + std::unique_ptr Apply() const override { + auto grad_op = new OpDescBind(); + grad_op->SetInput(GradVarName("Out"), OutputGrad("Out")); + grad_op->SetOutput(GradVarName("X"), InputGrad("X")); + grad_op->SetOutput(GradVarName("b"), InputGrad("b")); + grad_op->SetType("rowwise_add_grad"); + return std::unique_ptr(grad_op); } }; @@ -147,10 +147,8 @@ class SumOpMaker : public framework::OpProtoAndCheckerMaker { public: SumOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("X", "the input tensors of sum operator.") - .AsDuplicable() - .NotInGradient(); - AddOutput("Out", "the output tensor of sum operator.").NotInGradient(); + AddInput("X", "the input tensors of sum operator.").AsDuplicable(); + AddOutput("Out", "the output tensor of sum operator."); AddComment(""); } }; diff --git a/paddle/operators/mean_op.cc b/paddle/operators/mean_op.cc index 0c84cbb5a7..339c089e87 100644 --- a/paddle/operators/mean_op.cc +++ b/paddle/operators/mean_op.cc @@ -57,13 +57,13 @@ class MeanGradMaker : public framework::SingleGradOpDescMaker { using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; protected: - framework::OpDescBind Apply() const override { - framework::OpDescBind grad_op; - grad_op.SetType("mean_grad"); - grad_op.SetInput("X", Input("X")); - grad_op.SetInput(framework::GradVarName("Out"), OutputGrad("Out")); - grad_op.SetOutput(framework::GradVarName("X"), InputGrad("X")); - return grad_op; + std::unique_ptr Apply() const override { + auto* grad_op = new framework::OpDescBind(); + grad_op->SetType("mean_grad"); + grad_op->SetInput("X", Input("X")); + grad_op->SetInput(framework::GradVarName("Out"), OutputGrad("Out")); + grad_op->SetOutput(framework::GradVarName("X"), InputGrad("X")); + return std::unique_ptr(grad_op); } }; diff --git a/paddle/operators/minus_op.cc b/paddle/operators/minus_op.cc index 1b3ae9a9a6..aced8636b9 100644 --- a/paddle/operators/minus_op.cc +++ b/paddle/operators/minus_op.cc @@ -69,19 +69,22 @@ class MinusGradMaker : public framework::GradOpDescMakerBase { public: using framework::GradOpDescMakerBase::GradOpDescMakerBase; - std::vector operator()() const override { - std::vector ops; + std::vector> operator()() + const override { + std::vector> ops; ops.resize(2); - ops[0].SetType("scale"); - ops[0].SetInput("X", OutputGrad("Out")); - ops[0].SetOutput("Out", InputGrad("X")); - ops[0].SetAttr("scale", 1.0f); - - ops[1].SetType("scale"); - ops[1].SetInput("X", OutputGrad("Out")); - ops[1].SetOutput("Out", InputGrad("Y")); - ops[1].SetAttr("scale", -1.0f); + ops[0].reset(new framework::OpDescBind()); + ops[0]->SetType("scale"); + ops[0]->SetInput("X", OutputGrad("Out")); + ops[0]->SetOutput("Out", InputGrad("X")); + ops[0]->SetAttr("scale", 1.0f); + + ops[1].reset(new framework::OpDescBind()); + ops[1]->SetType("scale"); + ops[1]->SetInput("X", OutputGrad("Out")); + ops[1]->SetOutput("Out", InputGrad("Y")); + ops[1]->SetAttr("scale", -1.0f); return ops; } }; diff --git a/paddle/operators/pad_op.cc b/paddle/operators/pad_op.cc index 4bd25fa46a..9445917739 100644 --- a/paddle/operators/pad_op.cc +++ b/paddle/operators/pad_op.cc @@ -111,18 +111,18 @@ class PadOpGrad : public framework::OperatorWithKernel { }; class PadOpGradMaker : public framework::SingleGradOpDescMaker { - protected: - framework::OpDescBind Apply() const override { - framework::OpDescBind bind; - bind.SetInput("X", Input("X")); - bind.SetInput(framework::GradVarName("Out"), OutputGrad("Out")); - bind.SetOutput(framework::GradVarName("X"), InputGrad("X")); - bind.SetAttrMap(Attrs()); - return bind; - } - public: using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; + + protected: + std::unique_ptr Apply() const override { + auto* bind = new framework::OpDescBind(); + bind->SetInput("X", Input("X")); + bind->SetInput(framework::GradVarName("Out"), OutputGrad("Out")); + bind->SetOutput(framework::GradVarName("X"), InputGrad("X")); + bind->SetAttrMap(Attrs()); + return std::unique_ptr(bind); + } }; } // namespace operators diff --git a/paddle/operators/scale_op.cc b/paddle/operators/scale_op.cc index 40f0960923..e225aecc27 100644 --- a/paddle/operators/scale_op.cc +++ b/paddle/operators/scale_op.cc @@ -57,13 +57,13 @@ class ScaleGradMaker : public framework::SingleGradOpDescMaker { using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; protected: - framework::OpDescBind Apply() const override { - framework::OpDescBind grad_op; - grad_op.SetType("scale"); - grad_op.SetInput("X", OutputGrad("Out")); - grad_op.SetOutput("Out", InputGrad("X")); - grad_op.SetAttr("scale", GetAttr("scale")); - return grad_op; + std::unique_ptr Apply() const override { + auto *grad_op = new framework::OpDescBind(); + grad_op->SetType("scale"); + grad_op->SetInput("X", OutputGrad("Out")); + grad_op->SetOutput("Out", InputGrad("X")); + grad_op->SetAttr("scale", GetAttr("scale")); + return std::unique_ptr(grad_op); } }; diff --git a/paddle/operators/softmax_with_cross_entropy_op.cc b/paddle/operators/softmax_with_cross_entropy_op.cc index 87dcc3f240..bc9868874d 100644 --- a/paddle/operators/softmax_with_cross_entropy_op.cc +++ b/paddle/operators/softmax_with_cross_entropy_op.cc @@ -167,17 +167,17 @@ class SoftmaxGradMaker : public framework::SingleGradOpDescMaker { using framework::SingleGradOpDescMaker::SingleGradOpDescMaker; protected: - framework::OpDescBind Apply() const override { - framework::OpDescBind grad_op; - grad_op.SetType("softmax_with_cross_entropy_grad"); - grad_op.SetInput("Label", Input("Label")); - grad_op.SetInput("Softmax", Output("Softmax")); - grad_op.SetInput("Loss", Output("Loss")); - grad_op.SetInput(framework::GradVarName("Softmax"), OutputGrad("Softmax")); - grad_op.SetInput(framework::GradVarName("Loss"), OutputGrad("Loss")); - grad_op.SetOutput(framework::GradVarName("Logits"), InputGrad("Logits")); - grad_op.SetAttrMap(Attrs()); - return grad_op; + std::unique_ptr Apply() const override { + auto* grad_op = new framework::OpDescBind(); + grad_op->SetType("softmax_with_cross_entropy_grad"); + grad_op->SetInput("Label", Input("Label")); + grad_op->SetInput("Softmax", Output("Softmax")); + grad_op->SetInput("Loss", Output("Loss")); + grad_op->SetInput(framework::GradVarName("Softmax"), OutputGrad("Softmax")); + grad_op->SetInput(framework::GradVarName("Loss"), OutputGrad("Loss")); + grad_op->SetOutput(framework::GradVarName("Logits"), InputGrad("Logits")); + grad_op->SetAttrMap(Attrs()); + return std::unique_ptr(grad_op); } }; diff --git a/paddle/operators/sum_op.cc b/paddle/operators/sum_op.cc index 5ae13492b3..c701ee8dde 100644 --- a/paddle/operators/sum_op.cc +++ b/paddle/operators/sum_op.cc @@ -60,19 +60,20 @@ class SumGradMaker : public framework::GradOpDescMakerBase { public: using framework::GradOpDescMakerBase::GradOpDescMakerBase; - std::vector operator()() const override { + std::vector> operator()() + const override { auto x_grads = InputGrad("X"); - std::vector grad_ops; + std::vector> grad_ops; grad_ops.reserve(x_grads.size()); auto og = OutputGrad("Out"); std::transform(x_grads.begin(), x_grads.end(), std::back_inserter(grad_ops), [&og](const std::string& x_grad) { - framework::OpDescBind grad_op; - grad_op.SetType("scale"); - grad_op.SetInput("X", og); - grad_op.SetOutput("Out", {x_grad}); - grad_op.SetAttr("scale", 1.0f); - return grad_op; + auto* grad_op = new framework::OpDescBind(); + grad_op->SetType("scale"); + grad_op->SetInput("X", og); + grad_op->SetOutput("Out", {x_grad}); + grad_op->SetAttr("scale", 1.0f); + return std::unique_ptr(grad_op); }); return grad_ops; } From 94855f4af08002253fca10aab4bffc187e5c982f Mon Sep 17 00:00:00 2001 From: Kavya Srinet Date: Wed, 4 Oct 2017 12:45:56 -0700 Subject: [PATCH 134/170] Fixed changes proposed in the review --- paddle/operators/rmsprop_op.cc | 69 +++++++++++++------ paddle/operators/rmsprop_op.h | 19 +++-- .../v2/framework/tests/test_rmsprop_op.py | 24 ++++--- 3 files changed, 77 insertions(+), 35 deletions(-) diff --git a/paddle/operators/rmsprop_op.cc b/paddle/operators/rmsprop_op.cc index 602efab3db..1e06e08ede 100644 --- a/paddle/operators/rmsprop_op.cc +++ b/paddle/operators/rmsprop_op.cc @@ -25,25 +25,32 @@ class RmspropOp : public framework::OperatorWithKernel { void InferShape(framework::InferShapeContextBase *ctx) const override { PADDLE_ENFORCE(ctx->HasInput("Param"), "Input(Param) of RmspropOp should not be null."); + PADDLE_ENFORCE(ctx->HasInput("MeanSquare"), + "Input(MeanSquare) of RmspropOp should not be null."); + PADDLE_ENFORCE(ctx->HasInput("LearningRate"), + "Input(LearningRate) of RmspropOp should not be null."); PADDLE_ENFORCE(ctx->HasInput("Grad"), "Input(Grad) of RmspropOp should not be null."); PADDLE_ENFORCE(ctx->HasInput("Moment"), "Input(Moment) of RmspropOp should not be null."); - PADDLE_ENFORCE(ctx->HasInput("LearningRate"), - "Input(LearningRate) of RmspropOp should not be null."); PADDLE_ENFORCE(ctx->HasOutput("ParamOut"), "Output(param_out) of RmspropOp should not be null."); PADDLE_ENFORCE(ctx->HasOutput("MomentOut"), - "Output(moment_out) of RmspropOp should not be null."); + "Output(Momentum_out) of RmspropOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("MeanSquareOut"), + "Output(MeanSquareOut) of RmspropOp should not be null."); auto param_dim = ctx->GetInputDim("Param"); PADDLE_ENFORCE_EQ( param_dim, ctx->GetInputDim("Grad"), "Param and grad input of RmspropOp should have the same dimension."); - PADDLE_ENFORCE_EQ( - param_dim, ctx->GetInputDim("Moment"), - "Param and moment input of RmspropOp should have the same dimension."); + PADDLE_ENFORCE_EQ(param_dim, ctx->GetInputDim("Moment"), + "Param and Momentum input of RmspropOp " + "should have the same dimension."); + PADDLE_ENFORCE_EQ(param_dim, ctx->GetInputDim("MeanSquare"), + "Param and Momentum input of RmspropOp " + "should have the same dimension."); auto lr_dim = ctx->GetInputDim("LearningRate"); PADDLE_ENFORCE_EQ(framework::product(lr_dim), 1, @@ -51,6 +58,7 @@ class RmspropOp : public framework::OperatorWithKernel { ctx->SetOutputDim("ParamOut", param_dim); ctx->SetOutputDim("MomentOut", param_dim); + ctx->SetOutputDim("MeanSquareOut", param_dim); } }; @@ -59,27 +67,46 @@ class RmspropOpMaker : public framework::OpProtoAndCheckerMaker { RmspropOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("Param", "Input parameter"); - AddInput("Grad", "Input gradient"); - AddInput("Moment", "Second moment"); - AddInput("LearningRate", "Learning Rate"); - - AddOutput("ParamOut", "Output parameter"); - AddOutput("MomentOut", "Output second moment"); - - AddAttr("epsilon", "Constant for numerical stability"); - AddAttr("decayRate", "Decay rate for moving average of gradients"); + AddInput("Param", + "(Tensor, default Tensor) " + "Input parameter value that has to be updated"); + AddInput("MeanSquare", + "(Tensor, default Tensor)" + " The mean square value that gets updated"); + AddInput("LearningRate", + "(Tensor, default Tensor) " + "The learning rate should be a tensor of size 1"); + AddInput("Grad", + "(Tensor, default Tensor) " + "Input gradient of the parameter"); + AddInput("Moment", + "(Tensor, default Tensor) The moment that gets updated"); + + AddOutput("ParamOut", "(Tensor) Output updated parameter value"); + AddOutput("MomentOut", "(Tensor) Output updated moment"); + AddOutput("MeanSquareOut", "(Tensor) Output Mean squared updated value"); + + AddAttr("epsilon", + "(float, default 1e-10) Constant " + "for numerical stability.") + .SetDefault(1e-10); + AddAttr("decay", + "(float, default 0.9) " + "Discounting factor for coming gradient.") + .SetDefault(0.9); + AddAttr("momentum", "(float, default 0.0) Constant value") + .SetDefault(0.0); AddComment(R"DOC( RMSprop -MomentOut = decayRate * Moment + (1 - decayRate) * Grad * Grad -ParamOut = Param - LearningRate * Grad / (sqrt(MomentOut) + epsilon) +MeanSquareOut = decay * MeanSquare + (1 - decay) * Grad * Grad +MomentOut = momentum * Moment + + LearningRate * Grad / sqrt(MeanSquareOut + epsilon) +ParamOut = Param - MomentOut -The original slide(Slide 29 of +The original slides that proposed RMSprop: Slide 29 of http://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf) -does not have the epsilon attribute. It is added here for numerical stability -to avoid division by zero. )DOC"); } diff --git a/paddle/operators/rmsprop_op.h b/paddle/operators/rmsprop_op.h index 65b9edd35b..ed4b283ce4 100644 --- a/paddle/operators/rmsprop_op.h +++ b/paddle/operators/rmsprop_op.h @@ -30,23 +30,30 @@ class RmspropOpKernel : public framework::OpKernel { void Compute(const framework::ExecutionContext& ctx) const override { auto param_out = ctx.Output("ParamOut"); auto moment_out = ctx.Output("MomentOut"); + auto mean_square_out = ctx.Output("MeanSquareOut"); param_out->mutable_data(ctx.GetPlace()); moment_out->mutable_data(ctx.GetPlace()); + mean_square_out->mutable_data(ctx.GetPlace()); float epsilon = ctx.Attr("epsilon"); - float decay = ctx.Attr("decayRate"); + float rho = ctx.Attr("decay"); + float momentum = ctx.Attr("momentum"); auto p = EigenVector::Flatten(*ctx.Input("Param")); - auto g = EigenVector::Flatten(*ctx.Input("Grad")); - auto m = EigenVector::Flatten(*ctx.Input("Moment")); + auto ms = EigenVector::Flatten(*ctx.Input("MeanSquare")); float lr = ctx.Input("LearningRate")->data()[0]; + auto g = EigenVector::Flatten(*ctx.Input("Grad")); + auto mom = EigenVector::Flatten(*ctx.Input("Moment")); + auto p_out = EigenVector::Flatten(*param_out); - auto m_out = EigenVector::Flatten(*moment_out); + auto mom_out = EigenVector::Flatten(*moment_out); + auto ms_out = EigenVector::Flatten(*mean_square_out); auto place = ctx.GetEigenDevice(); - m_out.device(place) = decay * m + (1 - decay) * g * g; - p_out.device(place) = p - lr * g / (m_out.sqrt() + epsilon); + ms_out.device(place) = rho * ms + (1 - rho) * g * g; + mom_out.device(place) = momentum * mom + lr * g / (ms_out + epsilon).sqrt(); + p_out.device(place) = p - mom_out; } }; diff --git a/python/paddle/v2/framework/tests/test_rmsprop_op.py b/python/paddle/v2/framework/tests/test_rmsprop_op.py index 64ca5da48e..84bd815c8c 100644 --- a/python/paddle/v2/framework/tests/test_rmsprop_op.py +++ b/python/paddle/v2/framework/tests/test_rmsprop_op.py @@ -8,27 +8,35 @@ class TestRmspropOp(OpTest): self.op_type = "rmsprop" param = np.random.random((123, 321)).astype("float32") + mean_square = np.random.random((123, 321)).astype("float32") + learning_rate = np.array([0.01]).astype("float32") grad = np.random.random((123, 321)).astype("float32") moment = np.zeros((123, 321)).astype("float32") - learning_rate = np.array([0.01]).astype("float32") epsilon = 1e-6 - decay_rate = 0.9 + decay = 0.9 + momentum = 0.0 self.inputs = { 'Param': param, + 'MeanSquare': mean_square, + 'LearningRate': learning_rate, 'Grad': grad, 'Moment': moment, - 'LearningRate': learning_rate } - self.attrs = {'epsilon': epsilon, 'decayRate': decay_rate} + self.attrs = {'epsilon': epsilon, 'decay': decay, 'momentum': momentum} - moment_out = decay_rate * moment + (1 - decay_rate) * grad * grad - param_out = param - learning_rate * grad / (np.sqrt(moment_out) + - epsilon) + ms_out = decay * mean_square + (1 - decay) * grad * grad + moment_out = momentum * moment + \ + learning_rate * grad / np.sqrt(ms_out + epsilon) + param_out = param - moment_out - self.outputs = {'ParamOut': param_out, 'MomentOut': moment_out} + self.outputs = { + 'ParamOut': param_out, + 'MomentOut': moment_out, + 'MeanSquareOut': ms_out + } def test_check_output(self): self.check_output() From fa12e51675dbbc77eef75e5c346f2deecd45b0dc Mon Sep 17 00:00:00 2001 From: Kavya Srinet Date: Wed, 4 Oct 2017 13:27:40 -0700 Subject: [PATCH 135/170] Adding the default attribute test case --- paddle/operators/rmsprop_op.cc | 6 +-- paddle/operators/rmsprop_op.h | 6 +-- .../v2/framework/tests/test_rmsprop_op.py | 45 ++++++++++++++++++- 3 files changed, 50 insertions(+), 7 deletions(-) diff --git a/paddle/operators/rmsprop_op.cc b/paddle/operators/rmsprop_op.cc index 1e06e08ede..8f61c7fdda 100644 --- a/paddle/operators/rmsprop_op.cc +++ b/paddle/operators/rmsprop_op.cc @@ -89,13 +89,13 @@ class RmspropOpMaker : public framework::OpProtoAndCheckerMaker { AddAttr("epsilon", "(float, default 1e-10) Constant " "for numerical stability.") - .SetDefault(1e-10); + .SetDefault(1.0e-10f); AddAttr("decay", "(float, default 0.9) " "Discounting factor for coming gradient.") - .SetDefault(0.9); + .SetDefault(0.9f); AddAttr("momentum", "(float, default 0.0) Constant value") - .SetDefault(0.0); + .SetDefault(0.0f); AddComment(R"DOC( RMSprop diff --git a/paddle/operators/rmsprop_op.h b/paddle/operators/rmsprop_op.h index ed4b283ce4..9c04276ec6 100644 --- a/paddle/operators/rmsprop_op.h +++ b/paddle/operators/rmsprop_op.h @@ -28,9 +28,9 @@ template class RmspropOpKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { - auto param_out = ctx.Output("ParamOut"); - auto moment_out = ctx.Output("MomentOut"); - auto mean_square_out = ctx.Output("MeanSquareOut"); + auto* param_out = ctx.Output("ParamOut"); + auto* moment_out = ctx.Output("MomentOut"); + auto* mean_square_out = ctx.Output("MeanSquareOut"); param_out->mutable_data(ctx.GetPlace()); moment_out->mutable_data(ctx.GetPlace()); diff --git a/python/paddle/v2/framework/tests/test_rmsprop_op.py b/python/paddle/v2/framework/tests/test_rmsprop_op.py index 84bd815c8c..3e5ff733e9 100644 --- a/python/paddle/v2/framework/tests/test_rmsprop_op.py +++ b/python/paddle/v2/framework/tests/test_rmsprop_op.py @@ -3,7 +3,10 @@ import numpy as np from op_test import OpTest -class TestRmspropOp(OpTest): +class TestRmspropOp1(OpTest): + ''' Test RMSProp with explicit inputs + ''' + def setUp(self): self.op_type = "rmsprop" @@ -42,5 +45,45 @@ class TestRmspropOp(OpTest): self.check_output() +class TestRmspropOp2(OpTest): + '''Test RMSProp with defaukt values for attributes + ''' + + def setUp(self): + self.op_type = "rmsprop" + + param = np.random.random((123, 321)).astype("float32") + mean_square = np.random.random((123, 321)).astype("float32") + learning_rate = np.array([0.01]).astype("float32") + grad = np.random.random((123, 321)).astype("float32") + moment = np.zeros((123, 321)).astype("float32") + + epsilon = 1.0e-10 + decay = 0.9 + momentum = 0.0 + + self.inputs = { + 'Param': param, + 'MeanSquare': mean_square, + 'LearningRate': learning_rate, + 'Grad': grad, + 'Moment': moment, + } + + ms_out = decay * mean_square + (1 - decay) * grad * grad + moment_out = momentum * moment + \ + learning_rate * grad / np.sqrt(ms_out + epsilon) + param_out = param - moment_out + + self.outputs = { + 'ParamOut': param_out, + 'MomentOut': moment_out, + 'MeanSquareOut': ms_out + } + + def test_check_output(self): + self.check_output() + + if __name__ == "__main__": unittest.main() From a9e298bebef29390b815e431e1a475ab1417015a Mon Sep 17 00:00:00 2001 From: Helin Wang Date: Wed, 4 Oct 2017 13:40:32 -0700 Subject: [PATCH 136/170] fix according to comments --- doc/design/refactor/session.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/design/refactor/session.md b/doc/design/refactor/session.md index 9a7451ece5..1d9a26683c 100644 --- a/doc/design/refactor/session.md +++ b/doc/design/refactor/session.md @@ -86,10 +86,10 @@ Evaluates the target Operations or Variables in `targets`. OP's input as well: ```python - a = pd.constant(1.0, name="a") - b = pd.constant(2.0) + a = pd.constant(2.0, name="a") + b = pd.variable(name="b") c = pd.mul(a,b) - sess.eval(targets=c, feed_dict={"a":3.0}) # returns 6.0 + sess.eval(targets=c, feed_dict={"b":3.0}) # returns 6.0 ``` ```python @@ -107,14 +107,14 @@ session( ) ``` -Creates a new session. One session owns one scope, so creating +Creates a new session. One session owns one global scope, so creating multiple sessions will create different scopes. - *devices*: a single `string` or a list of `string` of device names, the corresponding devices will be the computation devices for `eval()`. If not specified, all available devices (e.g., all GPUs) will be used. The user doesn't need to specify the CPU device since - it will be always used. + it will be always used. Multiple sessions can use the same device. #### Example From 4558807c48035ee1d279d16975d6e2c607cb35f5 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Wed, 4 Oct 2017 14:01:34 -0700 Subject: [PATCH 137/170] Use PADDLE_WITH_CUDA instead of PADDLE_WITH_GPU --- paddle/api/Util.cpp | 2 +- paddle/capi/Matrix.cpp | 2 +- paddle/framework/grad_op_builder_test.cc | 2 +- paddle/framework/lod_tensor.h | 4 +-- paddle/framework/op_proto_maker_test.cc | 2 +- paddle/framework/op_registry.h | 2 +- paddle/framework/op_registry_test.cc | 2 +- paddle/framework/operator.cc | 2 +- paddle/framework/tensor_impl.h | 4 +-- paddle/framework/tensor_test.cc | 8 +++--- paddle/function/BlockExpandOp.cpp | 2 +- paddle/function/ContextProjectionOp.cpp | 2 +- paddle/function/CosSimOp.cpp | 2 +- paddle/function/CropOp.cpp | 2 +- paddle/function/CrossMapNormalOp.cpp | 2 +- paddle/function/DepthwiseConvOp.cpp | 2 +- paddle/function/DepthwiseConvOpTest.cpp | 2 +- paddle/function/GemmConvOp.cpp | 2 +- paddle/function/GemmConvOpTest.cpp | 2 +- paddle/function/Im2ColTest.cpp | 2 +- paddle/function/MulOp.cpp | 2 +- paddle/function/PadOp.cpp | 2 +- paddle/function/RowConvOp.cpp | 2 +- paddle/function/SwitchOp.cpp | 2 +- paddle/gserver/layers/BatchNormBaseLayer.cpp | 2 +- .../layers/BatchNormalizationLayer.cpp | 6 ++--- paddle/gserver/layers/PoolLayer.cpp | 4 +-- paddle/gserver/tests/LayerGradUtil.cpp | 2 +- paddle/gserver/tests/test_BatchNorm.cpp | 2 +- paddle/gserver/tests/test_ConvUnify.cpp | 2 +- paddle/gserver/tests/test_DetectionOutput.cpp | 2 +- paddle/gserver/tests/test_Evaluator.cpp | 2 +- paddle/gserver/tests/test_KmaxSeqScore.cpp | 2 +- paddle/gserver/tests/test_LayerGrad.cpp | 26 +++++++++---------- paddle/gserver/tests/test_NetworkCompare.cpp | 2 +- paddle/gserver/tests/test_PriorBox.cpp | 2 +- .../gserver/tests/test_ProtoDataProvider.cpp | 6 ++--- paddle/gserver/tests/test_PyDataProvider.cpp | 4 +-- .../gserver/tests/test_SelectiveFCLayer.cpp | 8 +++--- .../gserver/tests/test_SeqSliceLayerGrad.cpp | 2 +- paddle/gserver/tests/test_WarpCTCLayer.cpp | 2 +- paddle/math/Matrix.cpp | 6 ++--- paddle/math/SparseMatrix.cpp | 2 +- paddle/math/Vector.cpp | 6 ++--- paddle/math/tests/test_Allocator.cpp | 4 +-- paddle/math/tests/test_BaseMatrix.cpp | 2 +- paddle/math/tests/test_CpuGpuVector.cpp | 2 +- paddle/math/tests/test_ExecViaCpu.cpp | 2 +- paddle/math/tests/test_GpuProfiler.cpp | 2 +- paddle/math/tests/test_Matrix.cpp | 2 +- paddle/math/tests/test_SparseMatrix.cpp | 6 ++--- paddle/math/tests/test_TrainingAlgorithm.cpp | 2 +- paddle/math/tests/test_batchTranspose.cpp | 2 +- paddle/math/tests/test_matrixCompare.cpp | 2 +- paddle/math/tests/test_perturbation.cpp | 2 +- .../math/tests/test_sparseMatrixCompare.cpp | 2 +- paddle/memory/detail/buddy_allocator.cc | 2 +- paddle/memory/detail/system_allocator.cc | 2 +- paddle/memory/detail/system_allocator.h | 2 +- paddle/memory/detail/system_allocator_test.cc | 2 +- paddle/memory/memcpy.cc | 2 +- paddle/memory/memcpy.h | 2 +- paddle/memory/memory.cc | 2 +- paddle/memory/memory_test.cc | 2 +- paddle/operators/detail/strided_memcpy.h | 2 +- paddle/operators/math/im2col_test.cc | 4 +-- paddle/operators/math/math_function_test.cc | 2 +- paddle/operators/strided_memcpy_test.cc | 4 +-- paddle/platform/device_context.cc | 2 +- paddle/platform/device_context.h | 4 +-- paddle/platform/enforce.h | 4 +-- paddle/platform/enforce_test.cc | 2 +- paddle/platform/gpu_info.h | 2 +- paddle/platform/variant.h | 2 +- paddle/pserver/test/SocketTest.cpp | 2 +- paddle/pserver/test/test_ProtoServer.cpp | 2 +- paddle/pybind/pybind.cc | 12 ++++----- paddle/pybind/tensor_py.h | 2 +- paddle/string/to_string_test.cc | 2 +- paddle/trainer/MergeModel.cpp | 2 +- paddle/trainer/tests/test_Compare.cpp | 2 +- paddle/trainer/tests/test_CompareSparse.cpp | 4 +-- paddle/trainer/tests/test_Trainer.cpp | 4 +-- paddle/trainer/tests/test_TrainerOnePass.cpp | 6 ++--- .../test_recurrent_machine_generation.cpp | 2 +- paddle/utils/Flags.cpp | 2 +- paddle/utils/Util.h | 2 +- paddle/utils/Version.h | 2 +- 88 files changed, 134 insertions(+), 134 deletions(-) diff --git a/paddle/api/Util.cpp b/paddle/api/Util.cpp index 7446d892fd..11bd05c09d 100644 --- a/paddle/api/Util.cpp +++ b/paddle/api/Util.cpp @@ -47,7 +47,7 @@ bool isUsingGpu() { return FLAGS_use_gpu; } void setUseGpu(bool useGpu) { FLAGS_use_gpu = useGpu; } bool isGpuVersion() { -#ifndef PADDLE_WITH_GPU +#ifndef PADDLE_WITH_CUDA return false; #else return true; diff --git a/paddle/capi/Matrix.cpp b/paddle/capi/Matrix.cpp index 5b3737a759..4547afaf1d 100644 --- a/paddle/capi/Matrix.cpp +++ b/paddle/capi/Matrix.cpp @@ -46,7 +46,7 @@ paddle_error paddle_matrix_set_row(paddle_matrix mat, if (rowID >= ptr->mat->getHeight()) return kPD_OUT_OF_RANGE; paddle::real* buf = ptr->mat->getRowBuf(rowID); size_t width = ptr->mat->getWidth(); -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA hl_memcpy(buf, rowArray, sizeof(paddle::real) * width); #else std::copy(rowArray, rowArray + width, buf); diff --git a/paddle/framework/grad_op_builder_test.cc b/paddle/framework/grad_op_builder_test.cc index 2dbc2e6620..793780ea44 100644 --- a/paddle/framework/grad_op_builder_test.cc +++ b/paddle/framework/grad_op_builder_test.cc @@ -183,4 +183,4 @@ TEST(GradOpDescBuilder, IOIgnoredInGradient) { {f::GradVarName("in3_1"), f::GradVarName("in3_2")})); delete forw_op; delete grad_op; -} \ No newline at end of file +} diff --git a/paddle/framework/lod_tensor.h b/paddle/framework/lod_tensor.h index b12c95b6b7..4db36ee766 100644 --- a/paddle/framework/lod_tensor.h +++ b/paddle/framework/lod_tensor.h @@ -15,7 +15,7 @@ #pragma once #include -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA #include #include #include @@ -29,7 +29,7 @@ namespace paddle { namespace framework { -#ifndef PADDLE_WITH_GPU +#ifndef PADDLE_WITH_CUDA template using Vector = std::vector; #else diff --git a/paddle/framework/op_proto_maker_test.cc b/paddle/framework/op_proto_maker_test.cc index b01e30f753..988a14cf4d 100644 --- a/paddle/framework/op_proto_maker_test.cc +++ b/paddle/framework/op_proto_maker_test.cc @@ -48,4 +48,4 @@ TEST(ProtoMaker, DuplicatedInOut) { paddle::framework::OpAttrChecker op_checker; auto proto_maker = TestInOutProtoMaker(&op_proto, &op_checker); ASSERT_THROW(proto_maker.Validate(), paddle::platform::EnforceNotMet); -} \ No newline at end of file +} diff --git a/paddle/framework/op_registry.h b/paddle/framework/op_registry.h index aca6579f36..958cf581f5 100644 --- a/paddle/framework/op_registry.h +++ b/paddle/framework/op_registry.h @@ -211,7 +211,7 @@ class OpKernelRegistrar : public Registrar { // TODO(fengjiayi): The following macros // seems ugly, do we have better method? -#ifndef PADDLE_WITH_GPU +#ifndef PADDLE_WITH_CUDA #define USE_OP_KERNEL(op_type) USE_OP_DEVICE_KERNEL(op_type, CPU) #else #define USE_OP_KERNEL(op_type) \ diff --git a/paddle/framework/op_registry_test.cc b/paddle/framework/op_registry_test.cc index f89f40b444..b860fe6cac 100644 --- a/paddle/framework/op_registry_test.cc +++ b/paddle/framework/op_registry_test.cc @@ -183,4 +183,4 @@ class CosineOpComplete : public paddle::framework::CosineOp { TEST(OperatorRegistrar, Test) { using namespace paddle::framework; OperatorRegistrar reg("cos"); -} \ No newline at end of file +} diff --git a/paddle/framework/operator.cc b/paddle/framework/operator.cc index 21c1c6f9e6..2ca838f838 100644 --- a/paddle/framework/operator.cc +++ b/paddle/framework/operator.cc @@ -25,7 +25,7 @@ Eigen::DefaultDevice& ExecutionContext::GetEigenDevice< return *device_context_.GetEigenDevice(); } -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA template <> Eigen::GpuDevice& ExecutionContext::GetEigenDevice() const { diff --git a/paddle/framework/tensor_impl.h b/paddle/framework/tensor_impl.h index 1cde1f74b8..379eac94f9 100644 --- a/paddle/framework/tensor_impl.h +++ b/paddle/framework/tensor_impl.h @@ -65,7 +65,7 @@ inline T* Tensor::mutable_data(platform::Place place) { holder_.reset(new PlaceholderImpl( boost::get(place), size)); } else if (platform::is_gpu_place(place)) { -#ifndef PADDLE_WITH_GPU +#ifndef PADDLE_WITH_CUDA PADDLE_THROW("'GPUPlace' is not supported in CPU only device."); } #else @@ -103,7 +103,7 @@ inline void Tensor::CopyFrom(const Tensor& src, memory::Copy(boost::get(dst_place), dst_ptr, boost::get(src_place), src_ptr, size); } -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA else if (platform::is_gpu_place(src_place) && platform::is_cpu_place(dst_place)) { memory::Copy(boost::get(dst_place), dst_ptr, diff --git a/paddle/framework/tensor_test.cc b/paddle/framework/tensor_test.cc index 86c6945ab5..58cf0fc3cb 100644 --- a/paddle/framework/tensor_test.cc +++ b/paddle/framework/tensor_test.cc @@ -74,7 +74,7 @@ TEST(Tensor, MutableData) { EXPECT_EQ(p1, p2); } -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA { Tensor src_tensor; float* p1 = nullptr; @@ -126,7 +126,7 @@ TEST(Tensor, ShareDataWith) { ASSERT_EQ(src_tensor.data(), dst_tensor.data()); } -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA { Tensor src_tensor; Tensor dst_tensor; @@ -163,7 +163,7 @@ TEST(Tensor, Slice) { EXPECT_EQ(src_data_address + 3 * 4 * 1 * sizeof(int), slice_data_address); } -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA { Tensor src_tensor; src_tensor.mutable_data(make_ddim({6, 9}), GPUPlace()); @@ -218,7 +218,7 @@ TEST(Tensor, CopyFrom) { EXPECT_EQ(dst_ptr[i], slice_ptr[i]); } } -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA { Tensor src_tensor; Tensor gpu_tensor; diff --git a/paddle/function/BlockExpandOp.cpp b/paddle/function/BlockExpandOp.cpp index ad78f5f584..bd0fe119ce 100644 --- a/paddle/function/BlockExpandOp.cpp +++ b/paddle/function/BlockExpandOp.cpp @@ -194,7 +194,7 @@ public: REGISTER_TYPED_FUNC(BlockExpand, CPU, BlockExpandForward); REGISTER_TYPED_FUNC(BlockExpandGrad, CPU, BlockExpandBackward); -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA REGISTER_TYPED_FUNC(BlockExpand, GPU, BlockExpandForward); REGISTER_TYPED_FUNC(BlockExpandGrad, GPU, BlockExpandBackward); #endif diff --git a/paddle/function/ContextProjectionOp.cpp b/paddle/function/ContextProjectionOp.cpp index ab18c39df8..23916c0f4b 100644 --- a/paddle/function/ContextProjectionOp.cpp +++ b/paddle/function/ContextProjectionOp.cpp @@ -395,7 +395,7 @@ REGISTER_TYPED_FUNC(ContextProjectionForward, REGISTER_TYPED_FUNC(ContextProjectionBackward, CPU, ContextProjectionBackwardFunc); -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA REGISTER_TYPED_FUNC(ContextProjectionForward, GPU, ContextProjectionForwardFunc); diff --git a/paddle/function/CosSimOp.cpp b/paddle/function/CosSimOp.cpp index 4418f144d3..2e5c281f37 100644 --- a/paddle/function/CosSimOp.cpp +++ b/paddle/function/CosSimOp.cpp @@ -233,7 +233,7 @@ private: REGISTER_TYPED_FUNC(CosSimForward, CPU, CosSimForwardFunc); REGISTER_TYPED_FUNC(CosSimBackward, CPU, CosSimBackwardFunc); -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA REGISTER_TYPED_FUNC(CosSimForward, GPU, CosSimForwardFunc); REGISTER_TYPED_FUNC(CosSimBackward, GPU, CosSimBackwardFunc); #endif diff --git a/paddle/function/CropOp.cpp b/paddle/function/CropOp.cpp index 39504cc2c1..46f98f12c1 100644 --- a/paddle/function/CropOp.cpp +++ b/paddle/function/CropOp.cpp @@ -169,7 +169,7 @@ private: REGISTER_TYPED_FUNC(Crop, CPU, CropFunc); REGISTER_TYPED_FUNC(CropGrad, CPU, CropGradFunc); -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA REGISTER_TYPED_FUNC(Crop, GPU, CropFunc); REGISTER_TYPED_FUNC(CropGrad, GPU, CropGradFunc); #endif diff --git a/paddle/function/CrossMapNormalOp.cpp b/paddle/function/CrossMapNormalOp.cpp index 1cf0918bed..9e88669d37 100644 --- a/paddle/function/CrossMapNormalOp.cpp +++ b/paddle/function/CrossMapNormalOp.cpp @@ -336,7 +336,7 @@ private: REGISTER_TYPED_FUNC(CrossMapNormal, CPU, CrossMapNormalFunc); REGISTER_TYPED_FUNC(CrossMapNormalGrad, CPU, CrossMapNormalGradFunc); -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA REGISTER_TYPED_FUNC(CrossMapNormal, GPU, CrossMapNormalFunc); REGISTER_TYPED_FUNC(CrossMapNormalGrad, GPU, CrossMapNormalGradFunc); #endif diff --git a/paddle/function/DepthwiseConvOp.cpp b/paddle/function/DepthwiseConvOp.cpp index 7656ab3d0a..9863e3ae1d 100644 --- a/paddle/function/DepthwiseConvOp.cpp +++ b/paddle/function/DepthwiseConvOp.cpp @@ -292,7 +292,7 @@ REGISTER_TYPED_FUNC(DepthwiseConvGradInput, REGISTER_TYPED_FUNC(DepthwiseConvGradFilter, CPU, DepthwiseConvGradFilterFunction); -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA REGISTER_TYPED_FUNC(DepthwiseConv, GPU, DepthwiseConvFunction); REGISTER_TYPED_FUNC(DepthwiseConvGradInput, GPU, diff --git a/paddle/function/DepthwiseConvOpTest.cpp b/paddle/function/DepthwiseConvOpTest.cpp index 39033ecb2b..b1a90da7db 100644 --- a/paddle/function/DepthwiseConvOpTest.cpp +++ b/paddle/function/DepthwiseConvOpTest.cpp @@ -17,7 +17,7 @@ limitations under the License. */ namespace paddle { -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA TEST(DepthwiseConv, Forward) { DepthwiseConvolution( "GemmConv-CPU", "DepthwiseConv-GPU", forward); diff --git a/paddle/function/GemmConvOp.cpp b/paddle/function/GemmConvOp.cpp index 68e08c1480..bdb56ddac3 100644 --- a/paddle/function/GemmConvOp.cpp +++ b/paddle/function/GemmConvOp.cpp @@ -340,7 +340,7 @@ public: REGISTER_TYPED_FUNC(GemmConv, CPU, GemmConvFunction); REGISTER_TYPED_FUNC(GemmConvGradInput, CPU, GemmConvGradInputFunction); REGISTER_TYPED_FUNC(GemmConvGradFilter, CPU, GemmConvGradFilterFunction); -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA REGISTER_TYPED_FUNC(GemmConv, GPU, GemmConvFunction); REGISTER_TYPED_FUNC(GemmConvGradInput, GPU, GemmConvGradInputFunction); REGISTER_TYPED_FUNC(GemmConvGradFilter, GPU, GemmConvGradFilterFunction); diff --git a/paddle/function/GemmConvOpTest.cpp b/paddle/function/GemmConvOpTest.cpp index bd1cf3c6a4..b5b5e1f35b 100644 --- a/paddle/function/GemmConvOpTest.cpp +++ b/paddle/function/GemmConvOpTest.cpp @@ -24,7 +24,7 @@ TEST(GemmConv, NaiveConv) { "NaiveConv-CPU", "GemmConv-CPU", forward); } -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA TEST(GemmConv, Forward) { Convolution( "GemmConv-CPU", "GemmConv-GPU", forward); diff --git a/paddle/function/Im2ColTest.cpp b/paddle/function/Im2ColTest.cpp index 55325e94b5..a0a01a5fc7 100644 --- a/paddle/function/Im2ColTest.cpp +++ b/paddle/function/Im2ColTest.cpp @@ -116,7 +116,7 @@ void TestIm2ColFunctor() { TEST(Im2ColFunctor, CPU) { TestIm2ColFunctor(); } -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA TEST(Im2ColFunctor, GPU) { TestIm2ColFunctor(); } diff --git a/paddle/function/MulOp.cpp b/paddle/function/MulOp.cpp index 655026320c..704a8c4132 100644 --- a/paddle/function/MulOp.cpp +++ b/paddle/function/MulOp.cpp @@ -341,7 +341,7 @@ private: }; REGISTER_TYPED_FUNC(MulOp, CPU, MulFunc); -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA REGISTER_TYPED_FUNC(MulOp, GPU, MulFunc); #endif } // namespace paddle diff --git a/paddle/function/PadOp.cpp b/paddle/function/PadOp.cpp index 24c9bf4e72..eed2f2e308 100644 --- a/paddle/function/PadOp.cpp +++ b/paddle/function/PadOp.cpp @@ -207,7 +207,7 @@ private: REGISTER_TYPED_FUNC(Pad, CPU, PadFunc); REGISTER_TYPED_FUNC(PadGrad, CPU, PadGradFunc); -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA REGISTER_TYPED_FUNC(Pad, GPU, PadFunc); REGISTER_TYPED_FUNC(PadGrad, GPU, PadGradFunc); #endif diff --git a/paddle/function/RowConvOp.cpp b/paddle/function/RowConvOp.cpp index 09e702f71a..7c802d6627 100644 --- a/paddle/function/RowConvOp.cpp +++ b/paddle/function/RowConvOp.cpp @@ -217,7 +217,7 @@ public: REGISTER_TYPED_FUNC(RowConv, CPU, RowConvFunc); REGISTER_TYPED_FUNC(RowConvGrad, CPU, RowConvGradFunc); -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA REGISTER_TYPED_FUNC(RowConv, GPU, RowConvFunc); REGISTER_TYPED_FUNC(RowConvGrad, GPU, RowConvGradFunc); #endif diff --git a/paddle/function/SwitchOp.cpp b/paddle/function/SwitchOp.cpp index db839b5b76..597723a2dd 100644 --- a/paddle/function/SwitchOp.cpp +++ b/paddle/function/SwitchOp.cpp @@ -132,7 +132,7 @@ public: REGISTER_TYPED_FUNC(NCHW2NHWC, CPU, NCHW2NHWCFunc); REGISTER_TYPED_FUNC(NHWC2NCHW, CPU, NHWC2NCHWFunc); -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA REGISTER_TYPED_FUNC(NCHW2NHWC, GPU, NCHW2NHWCFunc); REGISTER_TYPED_FUNC(NHWC2NCHW, GPU, NHWC2NCHWFunc); #endif diff --git a/paddle/gserver/layers/BatchNormBaseLayer.cpp b/paddle/gserver/layers/BatchNormBaseLayer.cpp index 55f52816ab..bc7d1c83a4 100644 --- a/paddle/gserver/layers/BatchNormBaseLayer.cpp +++ b/paddle/gserver/layers/BatchNormBaseLayer.cpp @@ -16,7 +16,7 @@ limitations under the License. */ #include "BatchNormalizationLayer.h" #include "Layer.h" #include "paddle/utils/Stat.h" -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA #include "CudnnBatchNormLayer.h" #endif diff --git a/paddle/gserver/layers/BatchNormalizationLayer.cpp b/paddle/gserver/layers/BatchNormalizationLayer.cpp index 33cf24431d..dacff25e59 100644 --- a/paddle/gserver/layers/BatchNormalizationLayer.cpp +++ b/paddle/gserver/layers/BatchNormalizationLayer.cpp @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. */ #include "paddle/utils/Stat.h" -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA #include "hl_batch_transpose.h" #endif #include "BatchNormalizationLayer.h" @@ -90,7 +90,7 @@ void BatchNormalizationLayer::expandMat(const MatrixPtr& in, MatrixPtr& out) { size_t batchSize = in->getHeight(); CHECK_EQ(out->getHeight(), batchSize * imgPixels_); if (useGpu_) { -#ifndef PADDLE_WITH_GPU +#ifndef PADDLE_WITH_CUDA LOG(FATAL) << "paddle is compiled only for cpu"; #else batchTranspose( @@ -127,7 +127,7 @@ void BatchNormalizationLayer::shrinkMat(const MatrixPtr& in, MatrixPtr& out) { } CHECK_EQ(in->getHeight(), static_cast(batchSize * imgPixels_)); if (useGpu_) { -#ifndef PADDLE_WITH_GPU +#ifndef PADDLE_WITH_CUDA LOG(FATAL) << "paddle is compiled only for cpu"; #else batchTranspose( diff --git a/paddle/gserver/layers/PoolLayer.cpp b/paddle/gserver/layers/PoolLayer.cpp index 43ab4e4d47..7b932d5a76 100644 --- a/paddle/gserver/layers/PoolLayer.cpp +++ b/paddle/gserver/layers/PoolLayer.cpp @@ -15,7 +15,7 @@ limitations under the License. */ #include "PoolLayer.h" #include "PoolProjectionLayer.h" #include "paddle/utils/Logging.h" -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA #include "CudnnPoolLayer.h" #endif namespace paddle { @@ -53,7 +53,7 @@ Layer* PoolLayer::create(const LayerConfig& config) { const std::string& pool = config.inputs(0).pool_conf().pool_type(); if (pool == "max-projection" || pool == "avg-projection") { return new PoolProjectionLayer(config); -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA } else if (CudnnPoolLayer::typeCheck(pool)) { return new CudnnPoolLayer(config); #endif diff --git a/paddle/gserver/tests/LayerGradUtil.cpp b/paddle/gserver/tests/LayerGradUtil.cpp index 59df057a80..cd957c7c0b 100644 --- a/paddle/gserver/tests/LayerGradUtil.cpp +++ b/paddle/gserver/tests/LayerGradUtil.cpp @@ -674,7 +674,7 @@ void testLayerGradKernel(TestConfig testConf, bool useGpu, bool useWeight, float epsilon) { -#ifndef PADDLE_WITH_GPU +#ifndef PADDLE_WITH_CUDA if (useGpu) return; #endif FLAGS_use_gpu = useGpu; diff --git a/paddle/gserver/tests/test_BatchNorm.cpp b/paddle/gserver/tests/test_BatchNorm.cpp index c1c85f8fac..050fde9d0a 100644 --- a/paddle/gserver/tests/test_BatchNorm.cpp +++ b/paddle/gserver/tests/test_BatchNorm.cpp @@ -119,7 +119,7 @@ TEST(Layer, batchNorm) { CHECK_EQ(static_cast(convLayer->getOutputValue()->getWidth()), 576); } -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA void batchNormInference(int n, int c, int h, int w) { MatrixPtr input = std::make_shared(n, c * h * w); MatrixPtr cudnnOut = std::make_shared(n, c * h * w); diff --git a/paddle/gserver/tests/test_ConvUnify.cpp b/paddle/gserver/tests/test_ConvUnify.cpp index 16556469cb..ffcc47e2a8 100644 --- a/paddle/gserver/tests/test_ConvUnify.cpp +++ b/paddle/gserver/tests/test_ConvUnify.cpp @@ -117,7 +117,7 @@ MatrixPtr doOneConvTest(size_t imgSize, } TEST(Layer, convParaUnified) { -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA MatrixPtr input, resultCpu, resultGpu; /// TEST1 for conv /// diff --git a/paddle/gserver/tests/test_DetectionOutput.cpp b/paddle/gserver/tests/test_DetectionOutput.cpp index 1a83f48fae..dc39c97a87 100644 --- a/paddle/gserver/tests/test_DetectionOutput.cpp +++ b/paddle/gserver/tests/test_DetectionOutput.cpp @@ -150,7 +150,7 @@ TEST(Layer, detectionOutputLayerFwd) { useGpu, result2); -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA // GPU case 1. useGpu = true; inputLoc = Matrix::create(1, 16, false, useGpu); diff --git a/paddle/gserver/tests/test_Evaluator.cpp b/paddle/gserver/tests/test_Evaluator.cpp index 42bb570572..62a131171f 100644 --- a/paddle/gserver/tests/test_Evaluator.cpp +++ b/paddle/gserver/tests/test_Evaluator.cpp @@ -51,7 +51,7 @@ void testEvaluator(TestConfig testConf, string testEvaluatorName, size_t batchSize, bool useGpu) { -#ifndef PADDLE_WITH_GPU +#ifndef PADDLE_WITH_CUDA if (useGpu) return; #endif FLAGS_use_gpu = useGpu; diff --git a/paddle/gserver/tests/test_KmaxSeqScore.cpp b/paddle/gserver/tests/test_KmaxSeqScore.cpp index 1594de8502..6386259882 100644 --- a/paddle/gserver/tests/test_KmaxSeqScore.cpp +++ b/paddle/gserver/tests/test_KmaxSeqScore.cpp @@ -97,7 +97,7 @@ TEST(Layer, kmaxSeqScoreLayer) { Matrix::create(subSeqStartPosition.back(), 1, false, false); std::vector mode = {false}; -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA mode.push_back(true); #endif diff --git a/paddle/gserver/tests/test_LayerGrad.cpp b/paddle/gserver/tests/test_LayerGrad.cpp index e887dee5f9..90a3352898 100644 --- a/paddle/gserver/tests/test_LayerGrad.cpp +++ b/paddle/gserver/tests/test_LayerGrad.cpp @@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA #include #endif #include @@ -258,7 +258,7 @@ void testProjectionConv(size_t groups, bool isDeconv) { true); } -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA TEST(Projection, conv) { /// test ConvProjection testProjectionConv(1, false); @@ -422,7 +422,7 @@ TEST(Layer, depthwiseConvLayer) { // 'depthwise_conv' is a sepecial case of 'exconv' whose // groups size equals to the input channels size. testDepthwiseConvLayer("exconv", /* useGpu= */ false); -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA testDepthwiseConvLayer("exconv", /* useGpu= */ true); #endif } @@ -480,7 +480,7 @@ void testConvLayer(const string& type, bool trans, bool useGpu) { TEST(Layer, convLayer) { testConvLayer("exconv", /* trans= */ false, /* useGpu= */ false); -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA testConvLayer("exconv", /* trans= */ false, /* useGpu= */ true); testConvLayer("cudnn_conv", /* trans= */ false, /* useGpu= */ true); #endif @@ -525,7 +525,7 @@ TEST(Layer, convTransLayer) { for (auto useGpu : {false, true}) { testConvTransLayer("exconvt", /* trans= */ false, /* useGpu= */ useGpu); } -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA testConvTransLayer("cudnn_convt", /* trans= */ false, /* useGpu= */ true); #endif } @@ -638,7 +638,7 @@ TEST(Layer, SelectiveFullyConnectedLayer) { /* trans= */ false, /* useGup= */ false, false); -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA testLayerGrad(config, "selective_fc", 100, @@ -1210,7 +1210,7 @@ void testPoolLayer(const string& poolType, bool trans, bool useGpu) { testLayerGrad(config, "pool", 100, trans, useGpu); } -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA void testPoolLayer2(const string& poolType, bool trans, bool useGpu) { TestConfig config; config.inputDefs.push_back({INPUT_DATA, "layer_0", 3200, 0}); @@ -1236,7 +1236,7 @@ TEST(Layer, PoolLayer) { testPoolLayer("avg-projection", /* trans= */ false, /* useGpu= */ false); testPoolLayer("max-projection", /* trans= */ false, /* useGpu= */ false); -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA testPoolLayer("avg-projection", /* trans= */ false, /* useGpu= */ true); testPoolLayer("max-projection", /* trans= */ false, /* useGpu= */ true); testPoolLayer("cudnn-max-pool", /* trans= */ false, /* useGpu= */ true); @@ -1309,7 +1309,7 @@ void testPool3DLayer(const string& poolType, bool trans, bool useGpu) { TEST(Layer, Pool3DLayer) { testPool3DLayer("avg", /* trans= */ false, /* useGpu= */ false); testPool3DLayer("max", /* trans= */ false, /* useGpu= */ false); -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA testPool3DLayer("avg", /* trans= */ false, /* useGpu= */ true); testPool3DLayer("max", /* trans= */ false, /* useGpu= */ true); #endif @@ -1695,7 +1695,7 @@ void testBatchNormLayer(const string& type, bool trans, bool useGpu) { TEST(Layer, BatchNormalizationLayer) { testBatchNormLayer("batch_norm", false, false); -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA testBatchNormLayer("batch_norm", false, true); if (hl_get_cudnn_lib_version() >= int(4000)) { testBatchNormLayer("cudnn_batch_norm", false, true); @@ -1744,7 +1744,7 @@ void testBatchNorm3DLayer(const string& type, bool trans, bool useGpu) { TEST(Layer, testBatchNorm3DLayer) { testBatchNorm3DLayer("batch_norm", false, false); -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA testBatchNorm3DLayer("batch_norm", false, true); if (hl_get_cudnn_lib_version() >= int(4000)) { testBatchNorm3DLayer("cudnn_batch_norm", false, true); @@ -2262,7 +2262,7 @@ void test3DConvLayer(const string& type, bool trans, bool useGpu) { TEST(Layer, test3DConvLayer) { test3DConvLayer("conv3d", /* trans= */ false, /* useGpu= */ false); -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA test3DConvLayer("conv3d", /* trans= */ false, /* useGpu= */ true); #endif } @@ -2339,7 +2339,7 @@ void test3DDeConvLayer(const string& type, bool trans, bool useGpu) { TEST(Layer, test3DDeConvLayer) { test3DDeConvLayer("deconv3d", /* trans= */ false, /* useGpu= */ false); -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA test3DDeConvLayer("deconv3d", /* trans= */ false, /* useGpu= */ true); #endif } diff --git a/paddle/gserver/tests/test_NetworkCompare.cpp b/paddle/gserver/tests/test_NetworkCompare.cpp index e322fef9a4..2b92211936 100644 --- a/paddle/gserver/tests/test_NetworkCompare.cpp +++ b/paddle/gserver/tests/test_NetworkCompare.cpp @@ -243,7 +243,7 @@ TEST(Compare, concat_slice) { compareNetwork(config_file_a, config_file_b); } -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA TEST(Compare, img_pool) { std::string config_file_a = "./gserver/tests/img_pool_a.conf"; std::string config_file_b = "./gserver/tests/img_pool_b.conf"; diff --git a/paddle/gserver/tests/test_PriorBox.cpp b/paddle/gserver/tests/test_PriorBox.cpp index cbc0fff7b8..8dc5568784 100644 --- a/paddle/gserver/tests/test_PriorBox.cpp +++ b/paddle/gserver/tests/test_PriorBox.cpp @@ -151,7 +151,7 @@ TEST(Layer, priorBoxLayerFwd) { useGpu, result); -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA // reset the input parameters variance[1] = 0.1; variance[3] = 0.2; diff --git a/paddle/gserver/tests/test_ProtoDataProvider.cpp b/paddle/gserver/tests/test_ProtoDataProvider.cpp index 988dbc2513..af6472619d 100644 --- a/paddle/gserver/tests/test_ProtoDataProvider.cpp +++ b/paddle/gserver/tests/test_ProtoDataProvider.cpp @@ -485,7 +485,7 @@ TEST(ProtoDataProvider, test) { // Currently in async mode, useGpu is not supported continue; } -#ifndef PADDLE_WITH_GPU +#ifndef PADDLE_WITH_CUDA if (useGpu) { continue; } @@ -525,7 +525,7 @@ TEST(ProtoDataProvider, constant_slots) { for (int numConstantSlots : {1, 2}) { for (int useGpu : numTwoArray) { for (int dataCompression : numTwoArray) { -#ifndef PADDLE_WITH_GPU +#ifndef PADDLE_WITH_CUDA if (useGpu) { continue; } @@ -708,7 +708,7 @@ TEST(ProtoSequenceDataProvider, test) { // Currently in async mode, useGpu is not supported continue; } -#ifndef PADDLE_WITH_GPU +#ifndef PADDLE_WITH_CUDA if (useGpu) { continue; } diff --git a/paddle/gserver/tests/test_PyDataProvider.cpp b/paddle/gserver/tests/test_PyDataProvider.cpp index f6522febf8..fe54799259 100644 --- a/paddle/gserver/tests/test_PyDataProvider.cpp +++ b/paddle/gserver/tests/test_PyDataProvider.cpp @@ -37,7 +37,7 @@ TEST(PyDataProvider, py_fill_slots) { config.clear_files(); std::string dataFile = "gserver/tests/pyDataProvider/pyDataProviderList"; config.set_files(dataFile); -#ifndef PADDLE_WITH_GPU +#ifndef PADDLE_WITH_CUDA bool useGpu = false; #else bool useGpu = true; @@ -71,7 +71,7 @@ TEST(PyDataProvider, py_fill_nest_slots) { std::string dataFile = "gserver/tests/pyDataProvider/pyDataProviderList"; config.set_files(dataFile); EXPECT_EQ(config.IsInitialized(), true); -#ifndef PADDLE_WITH_GPU +#ifndef PADDLE_WITH_CUDA bool useGpu = false; #else bool useGpu = true; diff --git a/paddle/gserver/tests/test_SelectiveFCLayer.cpp b/paddle/gserver/tests/test_SelectiveFCLayer.cpp index b25d32fb2c..4c87fe1bba 100644 --- a/paddle/gserver/tests/test_SelectiveFCLayer.cpp +++ b/paddle/gserver/tests/test_SelectiveFCLayer.cpp @@ -321,7 +321,7 @@ TEST(Layer, SelectiveFcLayer_train_dense_mul) { "filelist=gserver/tests/SelectiveFcTest/dense_mul_list"; for (auto useGpu : {false, true}) { -#ifndef PADDLE_WITH_GPU +#ifndef PADDLE_WITH_CUDA if (useGpu) { break; } @@ -388,7 +388,7 @@ void testSelectiveFcLayerTrainSparseMul(const LayerConfig& config, outMatSelfc->getWidth(), outMatSelfc->getElementCnt())); cpuOutMatSelfc->copyFrom(*outMatSelfc, HPPL_STREAM_DEFAULT); -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA if (useGpu) { hl_stream_synchronize(HPPL_STREAM_DEFAULT); } @@ -418,7 +418,7 @@ void testSelectiveFcLayerTrainSparseMul(const LayerConfig& config, MatrixPtr cpuOutMatFc( new CpuMatrix(outMatFc->getHeight(), outMatFc->getWidth())); cpuOutMatFc->copyFrom(*outMatFc, HPPL_STREAM_DEFAULT); -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA if (useGpu) { hl_stream_synchronize(HPPL_STREAM_DEFAULT); } @@ -443,7 +443,7 @@ TEST(Layer, SelectiveFcLayer_train_sparse_mul) { selLayerConfig.set_size(fcLayerWidth); testSelectiveFcLayerTrainSparseMul(selLayerConfig, false); -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA testSelectiveFcLayerTrainSparseMul(selLayerConfig, true); #endif } diff --git a/paddle/gserver/tests/test_SeqSliceLayerGrad.cpp b/paddle/gserver/tests/test_SeqSliceLayerGrad.cpp index f28149081b..3366002ca1 100644 --- a/paddle/gserver/tests/test_SeqSliceLayerGrad.cpp +++ b/paddle/gserver/tests/test_SeqSliceLayerGrad.cpp @@ -195,7 +195,7 @@ TEST(Layer, SeqSliceLayer) { vector> ends; std::vector mode = {false}; -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA mode.push_back(true); #endif genSeqInfo(seqStartPos, subSeqStartPos); diff --git a/paddle/gserver/tests/test_WarpCTCLayer.cpp b/paddle/gserver/tests/test_WarpCTCLayer.cpp index ae5b64257f..da82946006 100644 --- a/paddle/gserver/tests/test_WarpCTCLayer.cpp +++ b/paddle/gserver/tests/test_WarpCTCLayer.cpp @@ -199,7 +199,7 @@ TEST(Layer, WarpCTCLayer) { for (auto batchSize : {1, 10, 32}) { for (auto normByTimes : {false, true}) { for (auto useGpu : {false, true}) { -#ifndef PADDLE_WITH_GPU +#ifndef PADDLE_WITH_CUDA if (useGpu) continue; #endif LOG(INFO) << "layerSize=" << layerSize << " batchSize=" << batchSize diff --git a/paddle/math/Matrix.cpp b/paddle/math/Matrix.cpp index de02f9c0d5..c3e34d5309 100644 --- a/paddle/math/Matrix.cpp +++ b/paddle/math/Matrix.cpp @@ -670,7 +670,7 @@ void GpuMatrix::leftMul(Matrix& a, real scaleAB, real scaleT) { } void GpuMatrix::selectRows(Matrix& table, IVector& ids) { -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA CHECK(dynamic_cast(&table)); CHECK(table.useGpu()); CHECK(ids.useGpu()); @@ -694,7 +694,7 @@ void GpuMatrix::selectRows(Matrix& table, IVector& ids) { } void GpuMatrix::addToRows(Matrix& table, IVector& ids) { -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA CHECK(dynamic_cast(&table)); CHECK(table.useGpu()); CHECK(ids.useGpu()); @@ -741,7 +741,7 @@ void GpuMatrix::rowMax(Matrix& max) { } void GpuMatrix::rowMax(IVector& maxIds, Matrix& maxVal) { -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA CHECK(maxIds.useGpu() && maxVal.useGpu()) << "Matrix type are not equal"; size_t numSamples = getHeight(); size_t beam = maxVal.getWidth(); diff --git a/paddle/math/SparseMatrix.cpp b/paddle/math/SparseMatrix.cpp index 1f31082ae8..284b68d590 100644 --- a/paddle/math/SparseMatrix.cpp +++ b/paddle/math/SparseMatrix.cpp @@ -836,7 +836,7 @@ void GpuSparseMatrix::zeroMem() { } void GpuSparseMatrix::rowMax(IVector& maxIds, Matrix& maxVal) { -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA CHECK(maxIds.useGpu() && maxVal.useGpu()) << "Matrix type are not equal"; size_t numSamples = getHeight(); size_t beam = maxVal.getWidth(); diff --git a/paddle/math/Vector.cpp b/paddle/math/Vector.cpp index 54e57b255d..ff72672e3a 100644 --- a/paddle/math/Vector.cpp +++ b/paddle/math/Vector.cpp @@ -172,7 +172,7 @@ void GpuVectorT::isEqualTo(const VectorT& b, const T& value) { template void GpuVectorT::selectFrom(const VectorT& src, const VectorT& ids) { -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA hl_vector_select_from(this->getData(), this->getSize(), src.getData(), @@ -850,7 +850,7 @@ CpuGpuVectorT::CpuGpuVectorT(CpuGpuVectorT& src, size_t size) : sync_(nullptr) { CHECK_LE(offset + size, static_cast(src.getSize())); -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA SyncedFlag* flag = src.getSync(); if (*flag == DATA_AT_CPU) { src.copyToGpu(); // will set synchronous data between CPU and GPU @@ -861,7 +861,7 @@ CpuGpuVectorT::CpuGpuVectorT(CpuGpuVectorT& src, auto cMemHandle = (src.getVector(false))->getMemoryHandle(); cpuVectorT_ = std::make_shared>( size, std::dynamic_pointer_cast(cMemHandle), offset); -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA auto gMemHandle = (src.getVector(true))->getMemoryHandle(); gpuVectorT_ = std::make_shared>( size, std::dynamic_pointer_cast(gMemHandle), offset); diff --git a/paddle/math/tests/test_Allocator.cpp b/paddle/math/tests/test_Allocator.cpp index cf2f66aea1..1fecf659e5 100644 --- a/paddle/math/tests/test_Allocator.cpp +++ b/paddle/math/tests/test_Allocator.cpp @@ -68,7 +68,7 @@ void testPoolAllocator() { TEST(Allocator, Pool) { testPoolAllocator(); -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA testPoolAllocator(); #endif } @@ -92,7 +92,7 @@ TEST(MemoryHandle, Cpu) { EXPECT_EQ(ptr1, ptr2); } -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA TEST(MemoryHandle, Gpu) { int numGpu = hl_get_device_count(); diff --git a/paddle/math/tests/test_BaseMatrix.cpp b/paddle/math/tests/test_BaseMatrix.cpp index 730759f3db..1766257860 100644 --- a/paddle/math/tests/test_BaseMatrix.cpp +++ b/paddle/math/tests/test_BaseMatrix.cpp @@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA /** * This test file use autotest::AutoCompare and cmpWithoutArg to compares the * implementation of CPU and GPU member function in diff --git a/paddle/math/tests/test_CpuGpuVector.cpp b/paddle/math/tests/test_CpuGpuVector.cpp index ccb4a902b0..c72f89c824 100644 --- a/paddle/math/tests/test_CpuGpuVector.cpp +++ b/paddle/math/tests/test_CpuGpuVector.cpp @@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA #include #include "paddle/math/Vector.h" diff --git a/paddle/math/tests/test_ExecViaCpu.cpp b/paddle/math/tests/test_ExecViaCpu.cpp index 2d439cd060..25e0ba11de 100644 --- a/paddle/math/tests/test_ExecViaCpu.cpp +++ b/paddle/math/tests/test_ExecViaCpu.cpp @@ -94,7 +94,7 @@ void testWrapper(F&& f) { } } -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA TEST(ExecViaCpu, test1) { testWrapper(f); testWrapper(&f); diff --git a/paddle/math/tests/test_GpuProfiler.cpp b/paddle/math/tests/test_GpuProfiler.cpp index 6dab187e3e..9402bd3ec4 100644 --- a/paddle/math/tests/test_GpuProfiler.cpp +++ b/paddle/math/tests/test_GpuProfiler.cpp @@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA #include #include "paddle/math/Matrix.h" diff --git a/paddle/math/tests/test_Matrix.cpp b/paddle/math/tests/test_Matrix.cpp index 7a145eae6a..2f99fa3581 100644 --- a/paddle/math/tests/test_Matrix.cpp +++ b/paddle/math/tests/test_Matrix.cpp @@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA /** * This test file use autotest::AutoCompare and cmpWithArg to compares the * implementation of CPU and GPU member function in Matrix.cpp. diff --git a/paddle/math/tests/test_SparseMatrix.cpp b/paddle/math/tests/test_SparseMatrix.cpp index 8151dde106..8abbe8d82e 100644 --- a/paddle/math/tests/test_SparseMatrix.cpp +++ b/paddle/math/tests/test_SparseMatrix.cpp @@ -47,7 +47,7 @@ struct MatrixPara { SparseFormat format; }; -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA void test_sparse_matrix_mul(MatrixPara paraA, MatrixPara paraB, MatrixPara paraC) { @@ -452,7 +452,7 @@ TEST(Matrix, SparseMatrixCSRFormatTrimFrom) { matB->trimFrom(*mat); checkSMatrixEqual2(matA, matB); -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA GpuSparseMatrixPtr matC = std::make_shared( height, trimedWidth, height, FLOAT_VALUE, SPARSE_CSR, true); matC->trimFrom(*mat); @@ -546,7 +546,7 @@ TEST(Matrix, SparseMatrixCSCFormatTrimFrom) { matB->trimFrom(*mat); checkSMatrixEqual2(matA, matB); -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA GpuSparseMatrixPtr matC = std::make_shared( height, trimedWidth, height, FLOAT_VALUE, SPARSE_CSC, true); matC->trimFrom(*mat); diff --git a/paddle/math/tests/test_TrainingAlgorithm.cpp b/paddle/math/tests/test_TrainingAlgorithm.cpp index 36ac024007..5ae0aa036f 100644 --- a/paddle/math/tests/test_TrainingAlgorithm.cpp +++ b/paddle/math/tests/test_TrainingAlgorithm.cpp @@ -91,7 +91,7 @@ int VectorCheckErr(const VectorPtr& vector1, const VectorPtr& vector2) { typedef std::function testMatrixFunc; void testCase(testMatrixFunc matrixFunc) { -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA for (auto useGpu : {false, true}) { #else for (auto useGpu : {false}) { diff --git a/paddle/math/tests/test_batchTranspose.cpp b/paddle/math/tests/test_batchTranspose.cpp index 0189e534eb..b70a619764 100644 --- a/paddle/math/tests/test_batchTranspose.cpp +++ b/paddle/math/tests/test_batchTranspose.cpp @@ -17,7 +17,7 @@ limitations under the License. */ using namespace paddle; // NOLINT -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA TEST(MatrixBatchTransTest, test_batch_matrix_transpose) { const int nx = 100; const int ny = 50; diff --git a/paddle/math/tests/test_matrixCompare.cpp b/paddle/math/tests/test_matrixCompare.cpp index 7735877ac8..7e5a1db44a 100644 --- a/paddle/math/tests/test_matrixCompare.cpp +++ b/paddle/math/tests/test_matrixCompare.cpp @@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA /// This unittest checks GpuMatrix/CpuMatrix get same result, so disable when /// only cpu version. diff --git a/paddle/math/tests/test_perturbation.cpp b/paddle/math/tests/test_perturbation.cpp index dff18136ae..c7c07c817a 100644 --- a/paddle/math/tests/test_perturbation.cpp +++ b/paddle/math/tests/test_perturbation.cpp @@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA #include #include diff --git a/paddle/math/tests/test_sparseMatrixCompare.cpp b/paddle/math/tests/test_sparseMatrixCompare.cpp index e39cc0a2f6..2b2a391b9d 100644 --- a/paddle/math/tests/test_sparseMatrixCompare.cpp +++ b/paddle/math/tests/test_sparseMatrixCompare.cpp @@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA /// This unittest checks GpuSparseMatrix/CpuSparseMatrix get same result, // so disable when /// only cpu version. diff --git a/paddle/memory/detail/buddy_allocator.cc b/paddle/memory/detail/buddy_allocator.cc index ed0c3374ff..fdc5ed19dc 100644 --- a/paddle/memory/detail/buddy_allocator.cc +++ b/paddle/memory/detail/buddy_allocator.cc @@ -175,7 +175,7 @@ void* BuddyAllocator::SystemAlloc(size_t size) { } BuddyAllocator::PoolSet::iterator BuddyAllocator::RefillPool() { -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA if (system_allocator_->UseGpu()) { if ((total_used_ + total_free_) == 0) { // Compute the maximum allocation size for the first allocation. diff --git a/paddle/memory/detail/system_allocator.cc b/paddle/memory/detail/system_allocator.cc index 64f8182b5c..6c9a46dd09 100644 --- a/paddle/memory/detail/system_allocator.cc +++ b/paddle/memory/detail/system_allocator.cc @@ -62,7 +62,7 @@ void CPUAllocator::Free(void* p, size_t size, size_t index) { bool CPUAllocator::UseGpu() const { return false; } -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA void* GPUAllocator::Alloc(size_t& index, size_t size) { // CUDA documentation doesn't explain if cudaMalloc returns nullptr diff --git a/paddle/memory/detail/system_allocator.h b/paddle/memory/detail/system_allocator.h index 6b1f40347b..ee9b012f91 100644 --- a/paddle/memory/detail/system_allocator.h +++ b/paddle/memory/detail/system_allocator.h @@ -40,7 +40,7 @@ class CPUAllocator : public SystemAllocator { virtual bool UseGpu() const; }; -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA class GPUAllocator : public SystemAllocator { public: virtual void* Alloc(size_t& index, size_t size); diff --git a/paddle/memory/detail/system_allocator_test.cc b/paddle/memory/detail/system_allocator_test.cc index 57d5443d50..cd563844e7 100644 --- a/paddle/memory/detail/system_allocator_test.cc +++ b/paddle/memory/detail/system_allocator_test.cc @@ -56,7 +56,7 @@ TEST(CPUAllocator, LockMem) { TestAllocator(a, 0); } -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA TEST(GPUAllocator, Alloc) { paddle::memory::detail::GPUAllocator a; TestAllocator(a, 2048); diff --git a/paddle/memory/memcpy.cc b/paddle/memory/memcpy.cc index 184d0f8fa7..790420a8ab 100644 --- a/paddle/memory/memcpy.cc +++ b/paddle/memory/memcpy.cc @@ -26,7 +26,7 @@ void Copy(platform::CPUPlace, void* dst, std::memcpy(dst, src, num); } -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA template <> void Copy(platform::CPUPlace dst_place, void* dst, diff --git a/paddle/memory/memcpy.h b/paddle/memory/memcpy.h index 7142831d43..0bccee58c3 100644 --- a/paddle/memory/memcpy.h +++ b/paddle/memory/memcpy.h @@ -33,7 +33,7 @@ namespace memory { template void Copy(DstPlace, void* dst, SrcPlace, const void* src, size_t num); -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA /** * \brief Copy memory from one place to another place. diff --git a/paddle/memory/memory.cc b/paddle/memory/memory.cc index 6d5a74dafe..355b6218d0 100644 --- a/paddle/memory/memory.cc +++ b/paddle/memory/memory.cc @@ -62,7 +62,7 @@ size_t Used(platform::CPUPlace place) { return GetCPUBuddyAllocator()->Used(); } -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA BuddyAllocator* GetGPUBuddyAllocator(int gpu_id) { using BuddyAllocVec = std::vector; diff --git a/paddle/memory/memory_test.cc b/paddle/memory/memory_test.cc index 7a617f04dc..0d402038a0 100644 --- a/paddle/memory/memory_test.cc +++ b/paddle/memory/memory_test.cc @@ -80,7 +80,7 @@ TEST(BuddyAllocator, CPUMultAlloc) { } } -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA size_t align(size_t size, paddle::platform::GPUPlace place) { size += sizeof(paddle::memory::detail::Metadata); diff --git a/paddle/operators/detail/strided_memcpy.h b/paddle/operators/detail/strided_memcpy.h index 9f05a26322..068c82f399 100644 --- a/paddle/operators/detail/strided_memcpy.h +++ b/paddle/operators/detail/strided_memcpy.h @@ -34,7 +34,7 @@ struct StridedMemcpyFunctor { auto& cpu_place = boost::get(place); memory::Copy(cpu_place, dst, cpu_place, src, sizeof(T) * dst_dim.head); } else { -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA auto& gpu_place = boost::get(place); auto& cuda_ctx = reinterpret_cast(dev_ctx); diff --git a/paddle/operators/math/im2col_test.cc b/paddle/operators/math/im2col_test.cc index 3d040ca2b5..40bdbfe733 100644 --- a/paddle/operators/math/im2col_test.cc +++ b/paddle/operators/math/im2col_test.cc @@ -71,7 +71,7 @@ void testIm2col() { context = new paddle::platform::CPUDeviceContext(paddle::platform::CPUPlace()); } else { -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA context = new paddle::platform::CUDADeviceContext(paddle::platform::GPUPlace()); #else @@ -116,7 +116,7 @@ void testIm2col() { TEST(math, im2col) { testIm2col(); -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA testIm2col(); #endif } diff --git a/paddle/operators/math/math_function_test.cc b/paddle/operators/math/math_function_test.cc index 2252268620..9945ba101d 100644 --- a/paddle/operators/math/math_function_test.cc +++ b/paddle/operators/math/math_function_test.cc @@ -1,7 +1,7 @@ #include "paddle/operators/math/math_function.h" #include "gtest/gtest.h" -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA TEST(math_function, notrans_mul_trans) { paddle::framework::Tensor input1; paddle::framework::Tensor input1_gpu; diff --git a/paddle/operators/strided_memcpy_test.cc b/paddle/operators/strided_memcpy_test.cc index e0dd7b19f1..68f064eaee 100644 --- a/paddle/operators/strided_memcpy_test.cc +++ b/paddle/operators/strided_memcpy_test.cc @@ -72,7 +72,7 @@ TEST(StridedMemcpy, CPUConcat) { } } -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA TEST(StridedMemcpy, GPUCrop) { // clang-format off int src[] = { @@ -157,4 +157,4 @@ TEST(StridedMemcpy, GPUConcat) { #endif } // namespace operators -} // namespace paddle \ No newline at end of file +} // namespace paddle diff --git a/paddle/platform/device_context.cc b/paddle/platform/device_context.cc index 8dcc357a16..a9b6b79903 100644 --- a/paddle/platform/device_context.cc +++ b/paddle/platform/device_context.cc @@ -35,7 +35,7 @@ Eigen::DefaultDevice* CPUDeviceContext::eigen_device() const { Place CPUDeviceContext::GetPlace() const { return CPUPlace(); } -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA template <> Eigen::GpuDevice* diff --git a/paddle/platform/device_context.h b/paddle/platform/device_context.h index c1c4c7f760..ef5f19214d 100644 --- a/paddle/platform/device_context.h +++ b/paddle/platform/device_context.h @@ -14,7 +14,7 @@ limitations under the License. */ #include "paddle/platform/enforce.h" #include "paddle/platform/place.h" -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA #include "paddle/platform/dynload/cublas.h" #include "paddle/platform/dynload/cudnn.h" #include "paddle/platform/gpu_info.h" @@ -61,7 +61,7 @@ class CPUDeviceContext : public DeviceContext { std::unique_ptr eigen_device_; }; -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA template <> struct EigenDeviceConverter { using EigenDeviceType = Eigen::GpuDevice; diff --git a/paddle/platform/enforce.h b/paddle/platform/enforce.h index f9fe521d50..15d8446cd8 100644 --- a/paddle/platform/enforce.h +++ b/paddle/platform/enforce.h @@ -29,7 +29,7 @@ limitations under the License. */ #include // for __cxa_demangle #endif -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA #include "paddle/platform/dynload/cublas.h" #include "paddle/platform/dynload/cudnn.h" @@ -113,7 +113,7 @@ inline typename std::enable_if::type throw_on_error( } } -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA template inline typename std::enable_if::type throw_on_error( diff --git a/paddle/platform/enforce_test.cc b/paddle/platform/enforce_test.cc index 80bdee3d9d..8206a055ea 100644 --- a/paddle/platform/enforce_test.cc +++ b/paddle/platform/enforce_test.cc @@ -213,4 +213,4 @@ TEST(ENFORCE_USER_DEFINED_CLASS, EQ) { TEST(ENFORCE_USER_DEFINED_CLASS, NE) { Dims a{{1, 2, 3, 4}}, b{{5, 6, 7, 8}}; ASSERT_THROW(PADDLE_ENFORCE_EQ(a, b), paddle::platform::EnforceNotMet); -} \ No newline at end of file +} diff --git a/paddle/platform/gpu_info.h b/paddle/platform/gpu_info.h index ac884386dd..e47c9b4a2a 100644 --- a/paddle/platform/gpu_info.h +++ b/paddle/platform/gpu_info.h @@ -14,7 +14,7 @@ limitations under the License. */ #pragma once -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA #include #include diff --git a/paddle/platform/variant.h b/paddle/platform/variant.h index 8145799dfd..619897ca19 100644 --- a/paddle/platform/variant.h +++ b/paddle/platform/variant.h @@ -16,7 +16,7 @@ #include -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA // Because boost's variadic templates has bug on nvcc, boost will disable // variadic template support when GPU enabled on nvcc. diff --git a/paddle/pserver/test/SocketTest.cpp b/paddle/pserver/test/SocketTest.cpp index 96724530f5..b43461d61b 100644 --- a/paddle/pserver/test/SocketTest.cpp +++ b/paddle/pserver/test/SocketTest.cpp @@ -215,7 +215,7 @@ int main(int argc, char** argv) { uint64_t dataSize = FLAGS_dim * sizeof(real); -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA GpuVector gpuParam(FLAGS_dim); GpuVector gpuGrad(FLAGS_dim); #else diff --git a/paddle/pserver/test/test_ProtoServer.cpp b/paddle/pserver/test/test_ProtoServer.cpp index 74ab1f2f77..ad8ffed9c1 100644 --- a/paddle/pserver/test/test_ProtoServer.cpp +++ b/paddle/pserver/test/test_ProtoServer.cpp @@ -99,7 +99,7 @@ TEST(ProtoServer, regular) { } TEST(ProtoServer, extended) { -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA ProtoClient* client; if (FLAGS_rdma_tcp == "rdma") client = new ProtoClient(FLAGS_server_addr, FLAGS_port, F_RDMA); diff --git a/paddle/pybind/pybind.cc b/paddle/pybind/pybind.cc index 761d82fc4d..cff54b1741 100644 --- a/paddle/pybind/pybind.cc +++ b/paddle/pybind/pybind.cc @@ -34,7 +34,7 @@ static size_t UniqueIntegerGenerator() { } bool IsCompileGPU() { -#ifndef PADDLE_WITH_GPU +#ifndef PADDLE_WITH_CUDA return false; #else return true; @@ -78,7 +78,7 @@ PYBIND11_PLUGIN(core) { .def("set", PyCPUTensorSetFromArray) .def("set", PyCPUTensorSetFromArray) .def("set", PyCPUTensorSetFromArray) -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA .def("set", PyCUDATensorSetFromArray) .def("set", PyCUDATensorSetFromArray) .def("set", PyCUDATensorSetFromArray) @@ -96,7 +96,7 @@ PYBIND11_PLUGIN(core) { .def( "__init__", [](LoDTensor &instance, const std::vector> &lod) { -#ifndef PADDLE_WITH_GPU +#ifndef PADDLE_WITH_CUDA new (&instance) LoDTensor(lod); #else LoD new_lod; @@ -107,7 +107,7 @@ PYBIND11_PLUGIN(core) { }) .def("set_lod", [](LoDTensor &self, const std::vector> &lod) { -#ifndef PADDLE_WITH_GPU +#ifndef PADDLE_WITH_CUDA self.set_lod(lod); #else LoD new_lod; @@ -117,7 +117,7 @@ PYBIND11_PLUGIN(core) { #endif }) .def("lod", [](LoDTensor &self) -> std::vector> { -#ifndef PADDLE_WITH_GPU +#ifndef PADDLE_WITH_CUDA return self.lod(); #else auto lod = self.lod(); @@ -203,7 +203,7 @@ All parameter, weight, gradient are variables in Paddle. .def_static("create", [](paddle::platform::GPUPlace& place) -> paddle::platform::DeviceContext* { -#ifndef PADDLE_WITH_GPU +#ifndef PADDLE_WITH_CUDA PADDLE_THROW("GPUPlace is not supported in CPU device."); #else return new paddle::platform::CUDADeviceContext(place); diff --git a/paddle/pybind/tensor_py.h b/paddle/pybind/tensor_py.h index 62e85fa54f..9e73f79cbd 100644 --- a/paddle/pybind/tensor_py.h +++ b/paddle/pybind/tensor_py.h @@ -106,7 +106,7 @@ void PyCPUTensorSetFromArray( std::memcpy(dst, array.data(), sizeof(T) * array.size()); } -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA template void PyCUDATensorSetFromArray( framework::Tensor &self, diff --git a/paddle/string/to_string_test.cc b/paddle/string/to_string_test.cc index 542c771a98..971484dd0c 100644 --- a/paddle/string/to_string_test.cc +++ b/paddle/string/to_string_test.cc @@ -36,4 +36,4 @@ TEST(to_string, user_defined) { using namespace paddle::string; UserDefinedClass instance; ASSERT_EQ(kOutputString, to_string(instance)); -} \ No newline at end of file +} diff --git a/paddle/trainer/MergeModel.cpp b/paddle/trainer/MergeModel.cpp index a37d53bc72..6c52eaf449 100644 --- a/paddle/trainer/MergeModel.cpp +++ b/paddle/trainer/MergeModel.cpp @@ -29,7 +29,7 @@ int main(int argc, char** argv) { initMain(argc, argv); initPython(argc, argv); string confFile = TrainerConfigHelper::getConfigNameFromPath(FLAGS_model_dir); -#ifndef PADDLE_WITH_GPU +#ifndef PADDLE_WITH_CUDA FLAGS_use_gpu = false; #endif auto config = std::make_shared(confFile); diff --git a/paddle/trainer/tests/test_Compare.cpp b/paddle/trainer/tests/test_Compare.cpp index b5d29da45a..f3a964acb6 100644 --- a/paddle/trainer/tests/test_Compare.cpp +++ b/paddle/trainer/tests/test_Compare.cpp @@ -146,7 +146,7 @@ void compareGradient(comData& comDataCpu, comData& comDataGpu) { } int main(int argc, char** argv) { -#ifndef PADDLE_WITH_GPU +#ifndef PADDLE_WITH_CUDA exit(0); #endif paddle::initMain(argc, argv); diff --git a/paddle/trainer/tests/test_CompareSparse.cpp b/paddle/trainer/tests/test_CompareSparse.cpp index 4da9ce20fb..5f1834bd73 100644 --- a/paddle/trainer/tests/test_CompareSparse.cpp +++ b/paddle/trainer/tests/test_CompareSparse.cpp @@ -174,7 +174,7 @@ TEST(compareSparse, multiGradientMachine) { FLAGS_local = local; FLAGS_ports_num_for_sparse = 5; for (bool useGpu : {false, true}) { -#ifndef PADDLE_WITH_GPU +#ifndef PADDLE_WITH_CUDA if (useGpu) continue; #endif FLAGS_parallel_nn = useGpu; @@ -198,7 +198,7 @@ TEST(compareSparse, NeuralNetwork) { FLAGS_local = local; FLAGS_ports_num_for_sparse = 5; for (bool useGpu : {false, true}) { -#ifndef PADDLE_WITH_GPU +#ifndef PADDLE_WITH_CUDA if (useGpu) continue; #endif FLAGS_parallel_nn = useGpu; diff --git a/paddle/trainer/tests/test_Trainer.cpp b/paddle/trainer/tests/test_Trainer.cpp index f69e1aafee..425b3d10a3 100644 --- a/paddle/trainer/tests/test_Trainer.cpp +++ b/paddle/trainer/tests/test_Trainer.cpp @@ -51,7 +51,7 @@ void checkGradientTest(const string& configFile, TEST(checkGradient, cpu) { checkGradientTest(configFile1, false, false); } -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA TEST(checkGradient, gpu) { checkGradientTest(configFile1, true, false); } TEST(checkGradient, multiGpu) { @@ -97,7 +97,7 @@ TEST(checkGradient, hsigmoid) { checkGradientTest(configFile2, false, false); } TEST(checkGradient, chunk) { checkGradientTest(configFile3, false, false); -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA checkGradientTest(configFile3, true, true); #endif } diff --git a/paddle/trainer/tests/test_TrainerOnePass.cpp b/paddle/trainer/tests/test_TrainerOnePass.cpp index 4c4d124fa9..b2a93d4d5e 100644 --- a/paddle/trainer/tests/test_TrainerOnePass.cpp +++ b/paddle/trainer/tests/test_TrainerOnePass.cpp @@ -79,7 +79,7 @@ void trainerOnePassTest(const string& configFile, // 1. test trainer (cpu, gpu). TEST(trainerOnePass, cpu) { trainerOnePassTest(configFile1, false, false); } -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA TEST(trainerOnePass, gpu) { trainerOnePassTest(configFile1, true, false); } TEST(trainerOnePass, gpu2) { trainerOnePassTest(configFile1, true, false, 2); } @@ -94,7 +94,7 @@ TEST(trainerOnePass, parallel) { #endif // 2. test average_window. -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA TEST(average_window, gpu) { trainerOnePassTest(configFile1, true, false, 4, 0.01); } @@ -266,7 +266,7 @@ TEST(checkRemoteUpdater, cpuTrainerOldUpdater) { checkRemoteParameterUpdaterTest(configFile1, false, false, 1, true); } -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA TEST(checkRemoteUpdater, gpuTrainer) { checkRemoteParameterUpdaterTest(configFile1, true, false); } diff --git a/paddle/trainer/tests/test_recurrent_machine_generation.cpp b/paddle/trainer/tests/test_recurrent_machine_generation.cpp index 74b4fed7ed..a8fbe31c2b 100644 --- a/paddle/trainer/tests/test_recurrent_machine_generation.cpp +++ b/paddle/trainer/tests/test_recurrent_machine_generation.cpp @@ -113,7 +113,7 @@ void testGeneration(const string& configFile, #ifndef PADDLE_TYPE_DOUBLE TEST(RecurrentGradientMachine, test_generation) { -#ifndef PADDLE_WITH_GPU +#ifndef PADDLE_WITH_CUDA const auto useGpuConfs = {false}; #else const auto useGpuConfs = {true, false}; diff --git a/paddle/utils/Flags.cpp b/paddle/utils/Flags.cpp index 32155ded35..8f100f02e9 100644 --- a/paddle/utils/Flags.cpp +++ b/paddle/utils/Flags.cpp @@ -14,7 +14,7 @@ limitations under the License. */ #include "Flags.h" -#ifndef PADDLE_WITH_GPU +#ifndef PADDLE_WITH_CUDA DEFINE_bool(use_gpu, false, "Only support CPU training"); #else DEFINE_bool(use_gpu, true, "Whether to use GPU for training"); diff --git a/paddle/utils/Util.h b/paddle/utils/Util.h index 904d0f5061..9579881ea3 100644 --- a/paddle/utils/Util.h +++ b/paddle/utils/Util.h @@ -218,7 +218,7 @@ protected: * *d2* is peer device to enable direct access to by the d1 device. */ inline void enablePeerAccess(int d1, int d2) { -#ifdef PADDLE_WITH_GPU +#ifdef PADDLE_WITH_CUDA if (hl_device_can_access_peer(d1, d2)) { SetDevice dev(d1); hl_device_enable_peer_access(d2); diff --git a/paddle/utils/Version.h b/paddle/utils/Version.h index 611fda83d9..004d62451c 100644 --- a/paddle/utils/Version.h +++ b/paddle/utils/Version.h @@ -48,7 +48,7 @@ void printVersion(std::ostream& os); * @return return true if paddle compiled with GPU */ constexpr bool isWithGpu() { -#ifndef PADDLE_WITH_GPU +#ifndef PADDLE_WITH_CUDA return false; #else return true; From 2b204f048bf6599bdb9ba799769404dc5fd206a8 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Wed, 4 Oct 2017 14:09:19 -0700 Subject: [PATCH 138/170] Rename platform::GetDeviceCount into platform::GetCUDADeviceCount --- paddle/memory/memory.cc | 2 +- paddle/platform/device_context_test.cc | 4 ++-- paddle/platform/gpu_info.cc | 4 ++-- paddle/platform/gpu_info.h | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/paddle/memory/memory.cc b/paddle/memory/memory.cc index 6d5a74dafe..f816962890 100644 --- a/paddle/memory/memory.cc +++ b/paddle/memory/memory.cc @@ -77,7 +77,7 @@ BuddyAllocator* GetGPUBuddyAllocator(int gpu_id) { // GPU buddy allocator initialization std::call_once(gpu_allocator_flag, [&]() { - int gpu_num = platform::GetDeviceCount(); + int gpu_num = platform::GetCUDADeviceCount(); allocators.reserve(gpu_num); for (int gpu = 0; gpu < gpu_num; gpu++) { platform::SetDeviceId(gpu); diff --git a/paddle/platform/device_context_test.cc b/paddle/platform/device_context_test.cc index f4b00c57de..8bf5174c4a 100644 --- a/paddle/platform/device_context_test.cc +++ b/paddle/platform/device_context_test.cc @@ -20,7 +20,7 @@ TEST(Device, Init) { using paddle::platform::CUDADeviceContext; using paddle::platform::GPUPlace; - int count = paddle::platform::GetDeviceCount(); + int count = paddle::platform::GetCUDADeviceCount(); for (int i = 0; i < count; i++) { DeviceContext* device_context = new CUDADeviceContext(GPUPlace(i)); Eigen::GpuDevice* gpu_device = @@ -34,7 +34,7 @@ TEST(Device, CUDADeviceContext) { using paddle::platform::CUDADeviceContext; using paddle::platform::GPUPlace; - int count = paddle::platform::GetDeviceCount(); + int count = paddle::platform::GetCUDADeviceCount(); for (int i = 0; i < count; i++) { CUDADeviceContext* device_context = new CUDADeviceContext(GPUPlace(i)); Eigen::GpuDevice* gpu_device = device_context->eigen_device(); diff --git a/paddle/platform/gpu_info.cc b/paddle/platform/gpu_info.cc index be381a4e26..70ad611d5d 100644 --- a/paddle/platform/gpu_info.cc +++ b/paddle/platform/gpu_info.cc @@ -26,11 +26,11 @@ DEFINE_double(fraction_of_gpu_memory_to_use, 0.95, namespace paddle { namespace platform { -int GetDeviceCount() { +int GetCUDADeviceCount() { int count; PADDLE_ENFORCE( cudaGetDeviceCount(&count), - "cudaGetDeviceCount failed in paddle::platform::GetDeviceCount"); + "cudaGetDeviceCount failed in paddle::platform::GetCUDADeviceCount"); return count; } diff --git a/paddle/platform/gpu_info.h b/paddle/platform/gpu_info.h index ac884386dd..276783bbe4 100644 --- a/paddle/platform/gpu_info.h +++ b/paddle/platform/gpu_info.h @@ -28,7 +28,7 @@ const std::string kEnvFractionGpuMemoryToUse = "PADDLE_FRACTION_GPU_MEMORY_TO_USE"; //! Get the total number of GPU devices in system. -int GetDeviceCount(); +int GetCUDADeviceCount(); //! Get the current GPU device id in system. int GetCurrentDeviceId(); From 5917e09cde86401005261914964eca4ef54de193 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Wed, 4 Oct 2017 15:26:19 -0700 Subject: [PATCH 139/170] tmp work --- paddle/framework/op_registry.h | 4 +++ paddle/framework/operator.h | 2 +- paddle/framework/shape_inference_map.cc | 9 ++++-- paddle/framework/shape_inference_map.h | 8 ----- paddle/pybind/pybind.cc | 9 ++++++ .../v2/framework/tests/test_infer_shape.py | 29 +++++++++++++++++++ 6 files changed, 49 insertions(+), 12 deletions(-) create mode 100644 python/paddle/v2/framework/tests/test_infer_shape.py diff --git a/paddle/framework/op_registry.h b/paddle/framework/op_registry.h index f04b6c503a..8138ba117a 100644 --- a/paddle/framework/op_registry.h +++ b/paddle/framework/op_registry.h @@ -55,6 +55,10 @@ class OpRegistry { const std::string& grad_op_type) { OperatorRegistrar reg(op_type.c_str()); reg.info.grad_op_type_ = grad_op_type; + auto proto = reg.info.Proto(); + std::cout << "====== " << op_type << " =======" << std::endl; + std::cout << proto.SerializeAsString() << std::endl; + std::cout << "=============" << std::endl; ShapeInferenceMap::Instance().CreateOpWithKernel(reg.info, op_type); // register gradient op if (!grad_op_type.empty()) { diff --git a/paddle/framework/operator.h b/paddle/framework/operator.h index 99f721cc67..458404af6d 100644 --- a/paddle/framework/operator.h +++ b/paddle/framework/operator.h @@ -598,9 +598,9 @@ class OperatorWithKernel : public OperatorBase { }); } - protected: virtual void InferShape(InferShapeContextBase* ctx) const = 0; + protected: // indicate kernel DataType by input data. Defaultly all input data must be // same. virtual DataType IndicateDataType(const ExecutionContext& ctx) const { diff --git a/paddle/framework/shape_inference_map.cc b/paddle/framework/shape_inference_map.cc index 1a27037221..bd2b867984 100644 --- a/paddle/framework/shape_inference_map.cc +++ b/paddle/framework/shape_inference_map.cc @@ -37,10 +37,13 @@ ShapeInferenceMap& ShapeInferenceMap::Instance() { void ShapeInferenceMap::CreateOpWithKernel(const OpInfo& op_info, const std::string& op_type) { - const VariableNameMap inputs = - ConvertOpProtoVarsToVarNameMap(op_info.Proto().inputs()); + auto proto = op_info.Proto(); + std::cout << "========= " << op_type << " in======" << std::endl; + std::cout << proto.SerializeAsString() << std::endl; + std::cout << "========= " << op_type << " out======" << std::endl; + const VariableNameMap inputs = ConvertOpProtoVarsToVarNameMap(proto.inputs()); const VariableNameMap outputs = - ConvertOpProtoVarsToVarNameMap(op_info.Proto().outputs()); + ConvertOpProtoVarsToVarNameMap(proto.outputs()); auto* op = op_info.Creator()(op_type, inputs, outputs, {}); auto* op_with_kernel = dynamic_cast(op); auto it = op_shape_inference_map_.find(op_type); diff --git a/paddle/framework/shape_inference_map.h b/paddle/framework/shape_inference_map.h index fb12669026..6c7304f6c0 100644 --- a/paddle/framework/shape_inference_map.h +++ b/paddle/framework/shape_inference_map.h @@ -27,14 +27,6 @@ class ShapeInferenceMap { public: static ShapeInferenceMap& Instance(); - const OperatorBase* GetOperator(const std::string& op_type) { - auto it = op_shape_inference_map_.find(op_type); - if (it == op_shape_inference_map_.end()) { - PADDLE_THROW("op with kernel for Op(%s) is not registered", op_type); - } - return it->second; - } - void CreateOpWithKernel(const OpInfo& op_info, const std::string& op_type); OperatorWithKernel* GetOpWithKernel(const std::string& op_type) { diff --git a/paddle/pybind/pybind.cc b/paddle/pybind/pybind.cc index f4121e9d71..e11bcc0e0f 100644 --- a/paddle/pybind/pybind.cc +++ b/paddle/pybind/pybind.cc @@ -223,6 +223,15 @@ All parameter, weight, gradient are variables in Paddle. desc.InitializationErrorString()); return OpRegistry::CreateOp(desc); }) + .def("infer_shape", + [](const OpDescBind &op_desc, BlockDescBind &block) { + auto &shape_inference_map = ShapeInferenceMap::Instance(); + auto *op = shape_inference_map.GetOpWithKernel(op_desc.Type()); + if (op != nullptr) { + auto ctx = CompileTimeInferShapeContext(op_desc, block); + op->InferShape(&ctx); + } + }) .def("backward", [](const OperatorBase &forwardOp, const std::unordered_set &no_grad_vars) { diff --git a/python/paddle/v2/framework/tests/test_infer_shape.py b/python/paddle/v2/framework/tests/test_infer_shape.py new file mode 100644 index 0000000000..56d3a90123 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_infer_shape.py @@ -0,0 +1,29 @@ +import unittest +import paddle.v2.framework.core as core +from paddle.v2.framework.op import Operator + + +class TestInferShape(unittest.TestCase): + def test_sum_op(self): + prog = core.ProgramDesc.__create_program_desc__() + self.assertIsNotNone(prog) + block = prog.block(0) + self.assertIsNotNone(block) + + # prepare input/output + x1 = block.new_var("x1") + x1.set_shape([10, 20]) + x2 = block.new_var("x2") + x2.set_shape([10, 20]) + + out = block.new_var("out") + + # prepare the operator + sum_op_desc = block.append_op() + sum_op_desc.set_type("sum") + sum_op_desc.set_input("X", ["x1", "x2"]) + sum_op_desc.set_output("Out", ["out"]) + + sum_op = Operator("sum", X=["x1", "x2"], Out="out") + sum_op.infer_shape(sum_op_desc, block) + print(out.shape()) From ee7b3ed09e699da191fe238ab409f32318637380 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Wed, 4 Oct 2017 15:33:44 -0700 Subject: [PATCH 140/170] use EigenScalar to get learning_rate from GPU device --- paddle/operators/sgd_op.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/paddle/operators/sgd_op.h b/paddle/operators/sgd_op.h index d72d333a9a..954fd48272 100644 --- a/paddle/operators/sgd_op.h +++ b/paddle/operators/sgd_op.h @@ -23,6 +23,9 @@ using Tensor = framework::Tensor; template using EigenVector = framework::EigenVector; +template +using EigenScalar = framework::EigenScalar; template class SGDOpKernel : public framework::OpKernel { @@ -31,13 +34,14 @@ class SGDOpKernel : public framework::OpKernel { auto param = ctx.Input("Param"); auto grad = ctx.Input("Grad"); auto param_out = ctx.Output("ParamOut"); - float lr = ctx.Input("LearningRate")->data()[0]; + auto learning_rate = ctx.Input("LearningRate"); param_out->mutable_data(ctx.GetPlace()); auto p = EigenVector::Flatten(*param); auto g = EigenVector::Flatten(*grad); auto o = EigenVector::Flatten(*param_out); + auto lr = EigenScalar::From(*learning_rate); auto place = ctx.GetEigenDevice(); o.device(place) = p - lr * g; From c4effc7d2d1225f15b1ec51ab54753d3ef693de7 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Wed, 4 Oct 2017 15:34:28 -0700 Subject: [PATCH 141/170] Fix CI Test --- paddle/framework/backward.cc | 3 +- paddle/framework/op_info.h | 7 +++- paddle/framework/op_registry.h | 34 +++++++++---------- paddle/operators/minus_op.cc | 33 +++++++++++------- paddle/operators/pad_op.cc | 1 + .../softmax_with_cross_entropy_op.cc | 9 +++-- 6 files changed, 52 insertions(+), 35 deletions(-) diff --git a/paddle/framework/backward.cc b/paddle/framework/backward.cc index 40390d4150..9193a1593e 100644 --- a/paddle/framework/backward.cc +++ b/paddle/framework/backward.cc @@ -33,7 +33,7 @@ static inline std::unique_ptr CreateGradOp( op_desc.SetType(op.Type()); op_desc.SetAttrMap(op.Attrs()); auto& info = OpInfoMap::Instance().Get(op.Type()); - auto grad_descs = info.grad_op_maker_(op_desc); + auto grad_descs = info.GradOpMaker()(op_desc); std::vector> grad_ops; grad_ops.reserve(grad_descs.size()); std::transform(grad_descs.begin(), grad_descs.end(), @@ -49,6 +49,7 @@ static inline std::unique_ptr CreateGradOp( for (auto& grad_op : grad_ops) { net_op->AppendOp(std::move(grad_op)); } + net_op->CompleteAddOp(); return std::unique_ptr(net_op); } } diff --git a/paddle/framework/op_info.h b/paddle/framework/op_info.h index 6f87e055b4..968f587b46 100644 --- a/paddle/framework/op_info.h +++ b/paddle/framework/op_info.h @@ -30,7 +30,6 @@ namespace framework { struct OpInfo { OpCreator creator_; - std::string grad_op_type_; GradOpMakerFN grad_op_maker_; OpProto* proto_{nullptr}; OpAttrChecker* checker_{nullptr}; @@ -51,6 +50,12 @@ struct OpInfo { "Operator Creator has not been registered"); return creator_; } + + const GradOpMakerFN& GradOpMaker() const { + PADDLE_ENFORCE_NOT_NULL(grad_op_maker_, + "Operator GradOpMaker has not been registered."); + return grad_op_maker_; + } }; class OpInfoMap { diff --git a/paddle/framework/op_registry.h b/paddle/framework/op_registry.h index da112fa488..a4f0144ce8 100644 --- a/paddle/framework/op_registry.h +++ b/paddle/framework/op_registry.h @@ -137,23 +137,21 @@ class OpKernelRegistrar : public Registrar { __test_global_namespace_##uniq_name##__>::value, \ msg) -#define VA_ARGS(...) , ##__VA_ARGS__ - -#define REGISTER_OPERATOR(op_type, op_class, ...) \ - STATIC_ASSERT_GLOBAL_NAMESPACE( \ - __reg_op__##op_type, \ - "REGISTER_OPERATOR must be called in global namespace"); \ - class _OpClass_##op_type##_ : public op_class { \ - public: \ - DEFINE_OP_CLONE_METHOD(_OpClass_##op_type##_); \ - DEFINE_OP_CONSTRUCTOR(_OpClass_##op_type##_, op_class); \ - }; \ - static ::paddle::framework::OperatorRegistrar<_OpClass_##op_type##_ VA_ARGS( \ - __VA_ARGS__)> \ - __op_registrar_##op_type##__(#op_type); \ - int TouchOpRegistrar_##op_type() { \ - __op_registrar_##op_type##__.Touch(); \ - return 0; \ +#define REGISTER_OPERATOR(op_type, op_class, ...) \ + STATIC_ASSERT_GLOBAL_NAMESPACE( \ + __reg_op__##op_type, \ + "REGISTER_OPERATOR must be called in global namespace"); \ + class _OpClass_##op_type##_ : public op_class { \ + public: \ + DEFINE_OP_CLONE_METHOD(_OpClass_##op_type##_); \ + DEFINE_OP_CONSTRUCTOR(_OpClass_##op_type##_, op_class); \ + }; \ + static ::paddle::framework::OperatorRegistrar<_OpClass_##op_type##_, \ + ##__VA_ARGS__> \ + __op_registrar_##op_type##__(#op_type); \ + int TouchOpRegistrar_##op_type() { \ + __op_registrar_##op_type##__.Touch(); \ + return 0; \ } /** @@ -170,7 +168,7 @@ class OpKernelRegistrar : public Registrar { virtual std::string GradOpType() const { return #grad_op_type; } \ }; \ REGISTER_OPERATOR(op_type, op_class, _GradOpDescMaker_##grad_op_type##_, \ - op_maker_class) + op_maker_class); #define REGISTER_OP_WITHOUT_GRADIENT(op_type, op_class, op_maker_class) \ REGISTER_OPERATOR(op_type, op_class, op_maker_class) diff --git a/paddle/operators/minus_op.cc b/paddle/operators/minus_op.cc index aced8636b9..7057dcbd6e 100644 --- a/paddle/operators/minus_op.cc +++ b/paddle/operators/minus_op.cc @@ -72,19 +72,26 @@ class MinusGradMaker : public framework::GradOpDescMakerBase { std::vector> operator()() const override { std::vector> ops; - ops.resize(2); - - ops[0].reset(new framework::OpDescBind()); - ops[0]->SetType("scale"); - ops[0]->SetInput("X", OutputGrad("Out")); - ops[0]->SetOutput("Out", InputGrad("X")); - ops[0]->SetAttr("scale", 1.0f); - - ops[1].reset(new framework::OpDescBind()); - ops[1]->SetType("scale"); - ops[1]->SetInput("X", OutputGrad("Out")); - ops[1]->SetOutput("Out", InputGrad("Y")); - ops[1]->SetAttr("scale", -1.0f); + auto x_g = InputGrad("X"); + if (!x_g.empty()) { + auto *x_g_op = new framework::OpDescBind(); + x_g_op->SetType("scale"); + x_g_op->SetInput("X", OutputGrad("Out")); + x_g_op->SetOutput("Out", x_g); + x_g_op->SetAttr("scale", 1.0f); + ops.emplace_back(x_g_op); + } + + auto y_g = InputGrad("Y"); + if (!y_g.empty()) { + auto *y_g_op = new framework::OpDescBind(); + y_g_op->SetType("scale"); + y_g_op->SetInput("X", OutputGrad("Out")); + y_g_op->SetOutput("Out", y_g); + y_g_op->SetAttr("scale", -1.0f); + ops.emplace_back(y_g_op); + } + return ops; } }; diff --git a/paddle/operators/pad_op.cc b/paddle/operators/pad_op.cc index 9445917739..15aa05f266 100644 --- a/paddle/operators/pad_op.cc +++ b/paddle/operators/pad_op.cc @@ -121,6 +121,7 @@ class PadOpGradMaker : public framework::SingleGradOpDescMaker { bind->SetInput(framework::GradVarName("Out"), OutputGrad("Out")); bind->SetOutput(framework::GradVarName("X"), InputGrad("X")); bind->SetAttrMap(Attrs()); + bind->SetType("pad_grad"); return std::unique_ptr(bind); } }; diff --git a/paddle/operators/softmax_with_cross_entropy_op.cc b/paddle/operators/softmax_with_cross_entropy_op.cc index bc9868874d..70fe429f59 100644 --- a/paddle/operators/softmax_with_cross_entropy_op.cc +++ b/paddle/operators/softmax_with_cross_entropy_op.cc @@ -14,6 +14,12 @@ #include "paddle/operators/softmax_with_cross_entropy_op.h" #include +#include + +#define DBG_LINE() \ + do { \ + std::cerr << "Run at " << __LINE__ << std::endl; \ + } while (false) namespace paddle { namespace operators { @@ -187,8 +193,7 @@ class SoftmaxGradMaker : public framework::SingleGradOpDescMaker { namespace ops = paddle::operators; REGISTER_OPERATOR(softmax_with_cross_entropy, ops::SoftmaxWithCrossEntropyOp, - ops::SoftmaxWithCrossEntropyOpMaker, - ops::SoftmaxWithCrossEntropyOpMaker); + ops::SoftmaxWithCrossEntropyOpMaker, ops::SoftmaxGradMaker); REGISTER_OPERATOR(softmax_with_cross_entropy_grad, ops::SoftmaxWithCrossEntropyOpGrad); REGISTER_OP_CPU_KERNEL(softmax_with_cross_entropy, From a270dbb778091d983010298b7972e05bcd6fe22e Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Wed, 4 Oct 2017 15:48:41 -0700 Subject: [PATCH 142/170] Add support for rnn_op --- paddle/framework/backward.cc | 53 +++++++++++++++++++++++++++-------- paddle/framework/block_desc.h | 9 ++++-- 2 files changed, 48 insertions(+), 14 deletions(-) diff --git a/paddle/framework/backward.cc b/paddle/framework/backward.cc index c4ede7d2fb..d9a42be5a2 100644 --- a/paddle/framework/backward.cc +++ b/paddle/framework/backward.cc @@ -20,6 +20,7 @@ #include "paddle/framework/block_desc.h" #include "paddle/framework/op_registry.h" +#include "paddle/framework/program_desc.h" #include "paddle/operators/net_op.h" #include "paddle/operators/recurrent_op.h" @@ -254,7 +255,7 @@ static bool AllGradInSet(const std::vector& names, return true; } -std::vector> MakeGradOpDescs( +std::vector> MakeOpGrad( const std::unique_ptr& op_desc, std::unordered_set& no_grad_vars) { std::vector> grad_op_descs; @@ -295,20 +296,35 @@ std::vector> MakeGradOpDescs( for (auto& p : pending_fill_zeros_ops) { grad_op_descs.push_back(std::move(p)); } - - // TODO(fengjiayi): RNN op return grad_op_descs; } -void AppendBackwardOpDescs(BlockDescBind& block_desc, - std::unordered_set& no_grad_vars) { +std::vector> MakeBlockBackward( + ProgramDescBind& program_desc, int block_idx, + std::unordered_set& no_grad_vars) { + BlockDescBind* cur_block = program_desc.Block(block_idx); + std::deque>& op_descs = cur_block->ops_; std::unordered_map> dup_out_ops; size_t grad_desc_idx = 0; - std::deque>& block_op_descs = block_desc.ops_; std::vector> backward_descs; - for (auto it = block_op_descs.rbegin(); it != block_op_descs.rend(); ++it) { + for (auto it = op_descs.rbegin(); it != op_descs.rend(); ++it) { std::vector> op_grads = - MakeGradOpDescs(*it, no_grad_vars); + MakeOpGrad(*it, no_grad_vars); + + if ((*it)->Type() == "recurrent") { + PADDLE_ENFORCE_EQ( + op_grads.size(), size_t(1), + "rnn_op's gradient process should contain only one op."); + int step_block_idx = (*it)->GetBlockAttr("stop_block"); + auto backward_block_op_descs = + MakeBlockBackward(program_desc, step_block_idx, no_grad_vars); + BlockDescBind* backward_block = program_desc.AppendBlock(*cur_block); + for (auto& ptr : backward_block_op_descs) { + backward_block->ops_.push_back(std::move(ptr)); + } + op_grads[0]->SetBlockAttr("step_block", *backward_block); + } + for (const auto& desc : op_grads) { for (const std::string& out_name : desc->OutputArgumentNames()) { dup_out_ops[out_name].emplace_back(grad_desc_idx); @@ -345,11 +361,24 @@ void AppendBackwardOpDescs(BlockDescBind& block_desc, backward_descs.insert(backward_descs.begin() + p.first + 1, std::move(p.second)); } - // Append backward_descs to BlockDescBind::ops_ - for (std::unique_ptr& ptr : backward_descs) { - block_op_descs.push_back(std::move(ptr)); + return backward_descs; +} + +void AppendBackward(ProgramDescBind& program_desc, + const std::unordered_set& no_grad_vars) { + std::unordered_set no_grad_var_names; + no_grad_var_names.reserve(no_grad_vars.size() + 1); + no_grad_var_names.insert(std::string(kEmptyVarName) + kGradVarSuffix); + for (auto& name : no_grad_vars) { + no_grad_var_names.insert(GradVarName(name)); + } + const int root_block_idx = 0; + auto backward_op_descs = + MakeBlockBackward(program_desc, root_block_idx, no_grad_var_names); + auto& forw_op_descs = program_desc.Block(root_block_idx)->ops_; + for (auto& ptr : backward_op_descs) { + forw_op_descs.push_back(std::move(ptr)); } - return; } } // namespace framework diff --git a/paddle/framework/block_desc.h b/paddle/framework/block_desc.h index fd95ef1901..aad1c3fef8 100644 --- a/paddle/framework/block_desc.h +++ b/paddle/framework/block_desc.h @@ -32,8 +32,13 @@ class ProgramDescBind; class BlockDescBind { public: - friend void AppendBackwardOpDescs( - BlockDescBind &block_desc, std::unordered_set &no_grad_vars); + friend std::vector> MakeBlockBackward( + ProgramDescBind &program_desc, int block_idx, + std::unordered_set &no_grad_vars); + + friend void AppendBackward( + ProgramDescBind &program_desc, + const std::unordered_set &no_grad_vars); BlockDescBind(ProgramDescBind *prog, BlockDesc *desc) : prog_(prog), desc_(desc), need_update_(false) {} From 2594a50245ae7bfe91f05d3c162af39b2457fa21 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Wed, 4 Oct 2017 15:49:43 -0700 Subject: [PATCH 143/170] Polish code --- paddle/framework/backward.cc | 1 - paddle/framework/backward_test.cc | 12 ------------ paddle/framework/op_info.h | 2 -- paddle/framework/op_registry.cc | 1 - paddle/operators/mean_op.cc | 1 - paddle/operators/softmax_with_cross_entropy_op.cc | 5 ----- 6 files changed, 22 deletions(-) diff --git a/paddle/framework/backward.cc b/paddle/framework/backward.cc index 9193a1593e..3d81dadfc4 100644 --- a/paddle/framework/backward.cc +++ b/paddle/framework/backward.cc @@ -203,7 +203,6 @@ static std::unique_ptr BackwardRecursive( } } else { std::unique_ptr grad_op(CreateGradOp(forwardOp)); - PADDLE_ENFORCE(grad_op != nullptr); ForEachVarName(grad_op->Inputs(), [&no_grad_names, &net, &grad_op]( const std::string& grad_input) { diff --git a/paddle/framework/backward_test.cc b/paddle/framework/backward_test.cc index 830d0427fa..a9b71cd809 100644 --- a/paddle/framework/backward_test.cc +++ b/paddle/framework/backward_test.cc @@ -171,17 +171,6 @@ REGISTER_OP_WITHOUT_GRADIENT(fc, f::FcOp, f::FcOpMaker); REGISTER_OP(many_output_op, f::NOP, f::ManyOutputOpMaker, many_output_op_grad, f::NOP); -// TEST(Backward, simple_op_grad) { -// auto fwd = f::OpRegistry::CreateOp( -// "rowwise_add", {{"X", {"x"}}, {"b", {"b"}}}, {{"Out", {"out"}}}, {}); -// ASSERT_NE(fwd, nullptr); -// auto gop = f::OpRegistry::CreateGradOp(*fwd); -// ASSERT_EQ(1UL, gop->Inputs().size()); -// ASSERT_EQ("rowwise_add_grad", gop->Type()); -// ASSERT_EQ(f::GradVarName("x"), gop->Output(f::GradVarName("X"))); -// ASSERT_EQ(f::GradVarName("b"), gop->Output(f::GradVarName("b"))); -//} - TEST(Backward, simple_op_not_need_grad) { auto fwd = f::OpRegistry::CreateOp( "rowwise_add", {{"X", {"x"}}, {"b", {"b"}}}, {{"Out", {"out"}}}, {}); @@ -390,7 +379,6 @@ TEST(Backward, linear_net_intermediate_variable_has_no_grad) { + 1UL /* external output number*/ + 1UL /* number of gradient of external output*/ + 2U /* internal variable number*/); - EXPECT_EQ(grad_fc.Outputs(all).size(), 2UL /* input number of mul*/ + 2UL /* input number of rowwise_add diff --git a/paddle/framework/op_info.h b/paddle/framework/op_info.h index 968f587b46..231f212fa3 100644 --- a/paddle/framework/op_info.h +++ b/paddle/framework/op_info.h @@ -23,8 +23,6 @@ #include "paddle/framework/type_defs.h" #include "paddle/platform/macros.h" -#include "glog/logging.h" - namespace paddle { namespace framework { diff --git a/paddle/framework/op_registry.cc b/paddle/framework/op_registry.cc index ac6aa8d28e..4dc83ec8fe 100644 --- a/paddle/framework/op_registry.cc +++ b/paddle/framework/op_registry.cc @@ -55,7 +55,6 @@ std::unique_ptr OpRegistry::CreateOp(const OpDesc& op_desc) { } std::unique_ptr OpRegistry::CreateOp(OpDescBind* op_desc) { - op_desc->Sync(); return CreateOp(op_desc->Type(), op_desc->Inputs(), op_desc->Outputs(), op_desc->GetAttrMap()); } diff --git a/paddle/operators/mean_op.cc b/paddle/operators/mean_op.cc index 339c089e87..2332c9546b 100644 --- a/paddle/operators/mean_op.cc +++ b/paddle/operators/mean_op.cc @@ -71,7 +71,6 @@ class MeanGradMaker : public framework::SingleGradOpDescMaker { } // namespace paddle namespace ops = paddle::operators; - REGISTER_OPERATOR(mean, ops::MeanOp, ops::MeanOpMaker, ops::MeanGradMaker); REGISTER_OPERATOR(mean_grad, ops::MeanGradOp); REGISTER_OP_CPU_KERNEL(mean, diff --git a/paddle/operators/softmax_with_cross_entropy_op.cc b/paddle/operators/softmax_with_cross_entropy_op.cc index 70fe429f59..42c1ba6fdf 100644 --- a/paddle/operators/softmax_with_cross_entropy_op.cc +++ b/paddle/operators/softmax_with_cross_entropy_op.cc @@ -16,11 +16,6 @@ #include #include -#define DBG_LINE() \ - do { \ - std::cerr << "Run at " << __LINE__ << std::endl; \ - } while (false) - namespace paddle { namespace operators { From ebbbaee0c3d69b26405875216dbc5e798adab4ba Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Wed, 4 Oct 2017 16:18:46 -0700 Subject: [PATCH 144/170] Follow comments --- paddle/framework/backward.cc | 4 ++-- paddle/framework/op_registry.cc | 6 +++--- paddle/framework/op_registry.h | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/paddle/framework/backward.cc b/paddle/framework/backward.cc index 3d81dadfc4..01ef385fd8 100644 --- a/paddle/framework/backward.cc +++ b/paddle/framework/backward.cc @@ -39,9 +39,9 @@ static inline std::unique_ptr CreateGradOp( std::transform(grad_descs.begin(), grad_descs.end(), std::back_inserter(grad_ops), [](const std::unique_ptr& grad_desc) { - return OpRegistry::CreateOp(grad_desc.get()); + return OpRegistry::CreateOp(*grad_desc); }); - PADDLE_ENFORCE_GT(grad_ops.size(), 0); + PADDLE_ENFORCE(!grad_ops.empty()); if (grad_ops.size() == 1) { return std::move(grad_ops[0]); } else { diff --git a/paddle/framework/op_registry.cc b/paddle/framework/op_registry.cc index 4dc83ec8fe..e9d2e55872 100644 --- a/paddle/framework/op_registry.cc +++ b/paddle/framework/op_registry.cc @@ -54,9 +54,9 @@ std::unique_ptr OpRegistry::CreateOp(const OpDesc& op_desc) { return CreateOp(op_desc.type(), inputs, outputs, attrs); } -std::unique_ptr OpRegistry::CreateOp(OpDescBind* op_desc) { - return CreateOp(op_desc->Type(), op_desc->Inputs(), op_desc->Outputs(), - op_desc->GetAttrMap()); +std::unique_ptr OpRegistry::CreateOp(const OpDescBind& op_desc) { + return CreateOp(op_desc.Type(), op_desc.Inputs(), op_desc.Outputs(), + op_desc.GetAttrMap()); } } // namespace framework diff --git a/paddle/framework/op_registry.h b/paddle/framework/op_registry.h index a4f0144ce8..4bf521c48d 100644 --- a/paddle/framework/op_registry.h +++ b/paddle/framework/op_registry.h @@ -79,7 +79,7 @@ class OpRegistry { static std::unique_ptr CreateOp(const OpDesc& op_desc); - static std::unique_ptr CreateOp(OpDescBind* op_desc); + static std::unique_ptr CreateOp(const OpDescBind& op_desc); }; template From cc1860c10ec15851a9bb9380f66ea6c8d4d144e2 Mon Sep 17 00:00:00 2001 From: Yu Yang Date: Wed, 4 Oct 2017 16:40:30 -0700 Subject: [PATCH 145/170] Add persistable in framework.proto --- paddle/framework/framework.proto | 1 + 1 file changed, 1 insertion(+) diff --git a/paddle/framework/framework.proto b/paddle/framework/framework.proto index 951c7afbc1..d696cdf5ad 100644 --- a/paddle/framework/framework.proto +++ b/paddle/framework/framework.proto @@ -106,6 +106,7 @@ message LoDTensorDesc { message VarDesc { required string name = 1; optional LoDTensorDesc lod_tensor = 2; + optional bool persistable = 3 [ default = false ]; } message BlockDesc { From 775c60246b66469e06f01a50c89b7b39594a3b63 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Wed, 4 Oct 2017 16:53:21 -0700 Subject: [PATCH 146/170] remove using in sgd header file --- paddle/operators/sgd_op.h | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/paddle/operators/sgd_op.h b/paddle/operators/sgd_op.h index 954fd48272..b501d244d7 100644 --- a/paddle/operators/sgd_op.h +++ b/paddle/operators/sgd_op.h @@ -19,32 +19,25 @@ limitations under the License. */ namespace paddle { namespace operators { -using Tensor = framework::Tensor; -template -using EigenVector = framework::EigenVector; -template -using EigenScalar = framework::EigenScalar; - template class SGDOpKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { - auto param = ctx.Input("Param"); - auto grad = ctx.Input("Grad"); - auto param_out = ctx.Output("ParamOut"); - auto learning_rate = ctx.Input("LearningRate"); + auto param = ctx.Input("Param"); + auto grad = ctx.Input("Grad"); + auto param_out = ctx.Output("ParamOut"); + auto learning_rate = ctx.Input("LearningRate"); param_out->mutable_data(ctx.GetPlace()); - auto p = EigenVector::Flatten(*param); - auto g = EigenVector::Flatten(*grad); - auto o = EigenVector::Flatten(*param_out); - auto lr = EigenScalar::From(*learning_rate); + auto p = framework::EigenVector::Flatten(*param); + auto g = framework::EigenVector::Flatten(*grad); + auto o = framework::EigenVector::Flatten(*param_out); + auto lr = framework::EigenVector::From(*learning_rate); auto place = ctx.GetEigenDevice(); - o.device(place) = p - lr * g; + Eigen::DSizes grad_dsize(grad->dims()[0], grad->dims()[1]); + o.device(place) = p - lr.broadcast(grad_dsize) * g; } }; From 8ebc31d9358c919fdd6f50d502f4ee071a91d38e Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Wed, 4 Oct 2017 17:13:02 -0700 Subject: [PATCH 147/170] optimize the dsize --- paddle/operators/sgd_op.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/paddle/operators/sgd_op.h b/paddle/operators/sgd_op.h index b501d244d7..26f4012f25 100644 --- a/paddle/operators/sgd_op.h +++ b/paddle/operators/sgd_op.h @@ -33,10 +33,10 @@ class SGDOpKernel : public framework::OpKernel { auto p = framework::EigenVector::Flatten(*param); auto g = framework::EigenVector::Flatten(*grad); auto o = framework::EigenVector::Flatten(*param_out); - auto lr = framework::EigenVector::From(*learning_rate); + auto lr = framework::EigenVector::Flatten(*learning_rate); auto place = ctx.GetEigenDevice(); - Eigen::DSizes grad_dsize(grad->dims()[0], grad->dims()[1]); + Eigen::DSizes grad_dsize(grad->numel()); o.device(place) = p - lr.broadcast(grad_dsize) * g; } }; From 46530f9e666012dd83c8763563f7acae8f5aad30 Mon Sep 17 00:00:00 2001 From: Kavya Srinet Date: Wed, 4 Oct 2017 19:02:22 -0700 Subject: [PATCH 148/170] Added Leaky Relu activation --- paddle/operators/activation_op.cc | 19 ++++++++++++ paddle/operators/activation_op.h | 30 ++++++++++++++++++- .../v2/framework/tests/test_activation_op.py | 17 +++++++++++ 3 files changed, 65 insertions(+), 1 deletion(-) diff --git a/paddle/operators/activation_op.cc b/paddle/operators/activation_op.cc index 7ae4d2f6b6..5f2ecc2673 100644 --- a/paddle/operators/activation_op.cc +++ b/paddle/operators/activation_op.cc @@ -69,6 +69,22 @@ class ReluOpMaker : public framework::OpProtoAndCheckerMaker { } }; +template +class LeakyReluOpMaker : public framework::OpProtoAndCheckerMaker { + public: + LeakyReluOpMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "Input of LeakyRelu operator"); + AddOutput("Y", "Output of LeakyRelu operator"); + AddComment( + "LeakyRelu activation operator, " + "leaky_relu = max(x, alpha * x)"); + AddAttr("alpha", "The small negative slope") + .SetDefault(static_cast(0.02f)); + } +}; + class TanhOpMaker : public framework::OpProtoAndCheckerMaker { public: TanhOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) @@ -240,6 +256,9 @@ REGISTER_OP(softsign, ops::ActivationOp, ops::SoftsignOpMaker, softsign_grad, REGISTER_OP(brelu, ops::ActivationOp, ops::BReluOpMaker, brelu_grad, ops::ActivationOpGrad); +REGISTER_OP(leaky_relu, ops::ActivationOp, ops::LeakyReluOpMaker, + leaky_relu_grad, ops::ActivationOpGrad); + REGISTER_OP(soft_relu, ops::ActivationOp, ops::SoftReluOpMaker, soft_relu_grad, ops::ActivationOpGrad); diff --git a/paddle/operators/activation_op.h b/paddle/operators/activation_op.h index ff35c2d97e..dae66cc77d 100644 --- a/paddle/operators/activation_op.h +++ b/paddle/operators/activation_op.h @@ -309,6 +309,33 @@ struct SoftReluGradFunctor : public BaseActivationFunctor { } }; +template +struct LeakyReluFunctor : public BaseActivationFunctor { + float alpha; + typename BaseActivationFunctor::AttrPair GetAttrs() { + return {{"alpha", &alpha}}; + } + + template + void operator()(Device d, X x, Y y) const { + y.device(d) = x.cwiseMax(alpha * x); + } +}; + +template +struct LeakyReluGradFunctor : public BaseActivationFunctor { + float alpha; + typename BaseActivationFunctor::AttrPair GetAttrs() { + return {{"alpha", &alpha}}; + } + template + void operator()(Device d, X x, Y y, dY dy, dX dx) const { + auto temp1 = alpha * (x < static_cast(0)).template cast().eval(); + auto temp2 = (x >= static_cast(0)).template cast().eval(); + dx.device(d) = dy * (temp1 + temp2).template cast(); + } +}; + template struct PowFunctor : public BaseActivationFunctor { float factor; @@ -379,4 +406,5 @@ struct STanhGradFunctor : public BaseActivationFunctor { __macro(soft_relu, SoftReluFunctor, SoftReluGradFunctor); \ __macro(pow, PowFunctor, PowGradFunctor); \ __macro(stanh, STanhFunctor, STanhGradFunctor); \ - __macro(softsign, SoftsignFunctor, SoftsignGradFunctor) + __macro(softsign, SoftsignFunctor, SoftsignGradFunctor); \ + __macro(leaky_relu, LeakyReluFunctor, LeakyReluGradFunctor) diff --git a/python/paddle/v2/framework/tests/test_activation_op.py b/python/paddle/v2/framework/tests/test_activation_op.py index c44eb84906..ce6dec7748 100644 --- a/python/paddle/v2/framework/tests/test_activation_op.py +++ b/python/paddle/v2/framework/tests/test_activation_op.py @@ -122,6 +122,23 @@ class TestBRelu(OpTest): self.check_grad(['X'], 'Y', max_relative_error=0.02) +class TestLeakyRelu(OpTest): + def setUp(self): + self.op_type = "leaky_relu" + alpha = 0.02 + self.attrs = {'alpha': alpha} + self.inputs = {'X': np.random.uniform(-3, 3, [4, 4]).astype("float32")} + self.outputs = { + 'Y': np.maximum(self.inputs['X'], alpha * self.inputs['X']) + } + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Y', max_relative_error=0.008) + + class TestSoftRelu(OpTest): def setUp(self): self.op_type = "soft_relu" From 47c994be07d608471734d5d14f980d83f2e0a7a6 Mon Sep 17 00:00:00 2001 From: Kavya Srinet Date: Thu, 5 Oct 2017 08:50:51 -0700 Subject: [PATCH 149/170] Updated the reltive error --- python/paddle/v2/framework/tests/test_activation_op.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/v2/framework/tests/test_activation_op.py b/python/paddle/v2/framework/tests/test_activation_op.py index ce6dec7748..f232996a55 100644 --- a/python/paddle/v2/framework/tests/test_activation_op.py +++ b/python/paddle/v2/framework/tests/test_activation_op.py @@ -136,7 +136,7 @@ class TestLeakyRelu(OpTest): self.check_output() def test_check_grad(self): - self.check_grad(['X'], 'Y', max_relative_error=0.008) + self.check_grad(['X'], 'Y', max_relative_error=0.007) class TestSoftRelu(OpTest): From 60af56c1b8e60240238d877c093bf9c99706fefe Mon Sep 17 00:00:00 2001 From: Kavya Srinet Date: Wed, 4 Oct 2017 19:02:22 -0700 Subject: [PATCH 150/170] Added Leaky Relu activation --- paddle/operators/activation_op.cc | 19 ++++++++++++ paddle/operators/activation_op.h | 30 ++++++++++++++++++- .../v2/framework/tests/test_activation_op.py | 17 +++++++++++ 3 files changed, 65 insertions(+), 1 deletion(-) diff --git a/paddle/operators/activation_op.cc b/paddle/operators/activation_op.cc index 7ae4d2f6b6..5f2ecc2673 100644 --- a/paddle/operators/activation_op.cc +++ b/paddle/operators/activation_op.cc @@ -69,6 +69,22 @@ class ReluOpMaker : public framework::OpProtoAndCheckerMaker { } }; +template +class LeakyReluOpMaker : public framework::OpProtoAndCheckerMaker { + public: + LeakyReluOpMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "Input of LeakyRelu operator"); + AddOutput("Y", "Output of LeakyRelu operator"); + AddComment( + "LeakyRelu activation operator, " + "leaky_relu = max(x, alpha * x)"); + AddAttr("alpha", "The small negative slope") + .SetDefault(static_cast(0.02f)); + } +}; + class TanhOpMaker : public framework::OpProtoAndCheckerMaker { public: TanhOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) @@ -240,6 +256,9 @@ REGISTER_OP(softsign, ops::ActivationOp, ops::SoftsignOpMaker, softsign_grad, REGISTER_OP(brelu, ops::ActivationOp, ops::BReluOpMaker, brelu_grad, ops::ActivationOpGrad); +REGISTER_OP(leaky_relu, ops::ActivationOp, ops::LeakyReluOpMaker, + leaky_relu_grad, ops::ActivationOpGrad); + REGISTER_OP(soft_relu, ops::ActivationOp, ops::SoftReluOpMaker, soft_relu_grad, ops::ActivationOpGrad); diff --git a/paddle/operators/activation_op.h b/paddle/operators/activation_op.h index ff35c2d97e..dae66cc77d 100644 --- a/paddle/operators/activation_op.h +++ b/paddle/operators/activation_op.h @@ -309,6 +309,33 @@ struct SoftReluGradFunctor : public BaseActivationFunctor { } }; +template +struct LeakyReluFunctor : public BaseActivationFunctor { + float alpha; + typename BaseActivationFunctor::AttrPair GetAttrs() { + return {{"alpha", &alpha}}; + } + + template + void operator()(Device d, X x, Y y) const { + y.device(d) = x.cwiseMax(alpha * x); + } +}; + +template +struct LeakyReluGradFunctor : public BaseActivationFunctor { + float alpha; + typename BaseActivationFunctor::AttrPair GetAttrs() { + return {{"alpha", &alpha}}; + } + template + void operator()(Device d, X x, Y y, dY dy, dX dx) const { + auto temp1 = alpha * (x < static_cast(0)).template cast().eval(); + auto temp2 = (x >= static_cast(0)).template cast().eval(); + dx.device(d) = dy * (temp1 + temp2).template cast(); + } +}; + template struct PowFunctor : public BaseActivationFunctor { float factor; @@ -379,4 +406,5 @@ struct STanhGradFunctor : public BaseActivationFunctor { __macro(soft_relu, SoftReluFunctor, SoftReluGradFunctor); \ __macro(pow, PowFunctor, PowGradFunctor); \ __macro(stanh, STanhFunctor, STanhGradFunctor); \ - __macro(softsign, SoftsignFunctor, SoftsignGradFunctor) + __macro(softsign, SoftsignFunctor, SoftsignGradFunctor); \ + __macro(leaky_relu, LeakyReluFunctor, LeakyReluGradFunctor) diff --git a/python/paddle/v2/framework/tests/test_activation_op.py b/python/paddle/v2/framework/tests/test_activation_op.py index c44eb84906..ce6dec7748 100644 --- a/python/paddle/v2/framework/tests/test_activation_op.py +++ b/python/paddle/v2/framework/tests/test_activation_op.py @@ -122,6 +122,23 @@ class TestBRelu(OpTest): self.check_grad(['X'], 'Y', max_relative_error=0.02) +class TestLeakyRelu(OpTest): + def setUp(self): + self.op_type = "leaky_relu" + alpha = 0.02 + self.attrs = {'alpha': alpha} + self.inputs = {'X': np.random.uniform(-3, 3, [4, 4]).astype("float32")} + self.outputs = { + 'Y': np.maximum(self.inputs['X'], alpha * self.inputs['X']) + } + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Y', max_relative_error=0.008) + + class TestSoftRelu(OpTest): def setUp(self): self.op_type = "soft_relu" From 11070e5f36be24be8da6fa2b70a2dae13212c513 Mon Sep 17 00:00:00 2001 From: Kavya Srinet Date: Thu, 5 Oct 2017 08:50:51 -0700 Subject: [PATCH 151/170] Updated the reltive error --- python/paddle/v2/framework/tests/test_activation_op.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/v2/framework/tests/test_activation_op.py b/python/paddle/v2/framework/tests/test_activation_op.py index ce6dec7748..f232996a55 100644 --- a/python/paddle/v2/framework/tests/test_activation_op.py +++ b/python/paddle/v2/framework/tests/test_activation_op.py @@ -136,7 +136,7 @@ class TestLeakyRelu(OpTest): self.check_output() def test_check_grad(self): - self.check_grad(['X'], 'Y', max_relative_error=0.008) + self.check_grad(['X'], 'Y', max_relative_error=0.007) class TestSoftRelu(OpTest): From 828c5b3e1dd3c80b955a1c65179ca6d5a27d852d Mon Sep 17 00:00:00 2001 From: Abhinav Arora Date: Thu, 5 Oct 2017 13:07:55 -0700 Subject: [PATCH 152/170] Adding Adadelta optimization operator (#4576) * Adding Adadelta optimization operator * Making inputs and outputs conform to naming convention * Removing type alias from header files * Fixing Adadelta documentation in comments * Addressing code review feedback --- paddle/operators/adadelta_op.cc | 115 ++++++++++++++++++ paddle/operators/adadelta_op.cu | 20 +++ paddle/operators/adadelta_op.h | 69 +++++++++++ .../v2/framework/tests/test_adadelta_op.py | 96 +++++++++++++++ 4 files changed, 300 insertions(+) create mode 100644 paddle/operators/adadelta_op.cc create mode 100644 paddle/operators/adadelta_op.cu create mode 100644 paddle/operators/adadelta_op.h create mode 100644 python/paddle/v2/framework/tests/test_adadelta_op.py diff --git a/paddle/operators/adadelta_op.cc b/paddle/operators/adadelta_op.cc new file mode 100644 index 0000000000..bd8c93b4a1 --- /dev/null +++ b/paddle/operators/adadelta_op.cc @@ -0,0 +1,115 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "paddle/operators/adadelta_op.h" + +namespace paddle { +namespace operators { + +class AdadeltaOp : public framework::OperatorWithKernel { + public: + using framework::OperatorWithKernel::OperatorWithKernel; + + protected: + void InferShape(framework::InferShapeContextBase *ctx) const override { + PADDLE_ENFORCE(ctx->HasInput("Param"), + "Input(Param) of AdadeltaOp should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Grad"), + "Input(Grad) of AdadeltaOp should not be null."); + PADDLE_ENFORCE(ctx->HasInput("AvgSquaredGrad"), + "Input(AvgSquaredGrad) of AdadeltaOp should not be null."); + PADDLE_ENFORCE(ctx->HasInput("AvgSquaredUpdate"), + "Input(AvgSquaredUpdate) of AdadeltaOp should not be null."); + + PADDLE_ENFORCE(ctx->HasOutput("ParamOut"), + "Output(ParamOut) of AdadeltaOp should not be null."); + PADDLE_ENFORCE( + ctx->HasOutput("AvgSquaredGradOut"), + "Output(AvgSquaredGradOut) of AdadeltaOp should not be null."); + PADDLE_ENFORCE( + ctx->HasOutput("AvgSquaredUpdateOut"), + "Output(AvgSquaredUpdateOut) of AdadeltaOp should not be null."); + + auto param_dim = ctx->GetInputDim("Param"); + PADDLE_ENFORCE_EQ( + param_dim, ctx->GetInputDim("Grad"), + "param and grad input of AdadeltaOp should have same dimension"); + PADDLE_ENFORCE_EQ(param_dim, ctx->GetInputDim("AvgSquaredGrad"), + "Param and AvgSquaredGrad input of AdadeltaOp " + "should have same dimension"); + PADDLE_ENFORCE_EQ(param_dim, ctx->GetInputDim("AvgSquaredUpdate"), + "Param and AvgSquaredUpdate input of AdadeltaOp " + "should have same dimension"); + + ctx->SetOutputDim("ParamOut", param_dim); + ctx->SetOutputDim("AvgSquaredGradOut", param_dim); + ctx->SetOutputDim("AvgSquaredUpdateOut", param_dim); + } +}; + +class AdadeltaOpMaker : public framework::OpProtoAndCheckerMaker { + public: + AdadeltaOpMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("Param", "(Tensor) Input parameter"); + AddInput("Grad", "(Tensor) Input gradient"); + AddInput("AvgSquaredGrad", + "(Tensor) Input expectation of squared gradient"); + AddInput("AvgSquaredUpdate", + "(Tensor) Input expectation of squared parameter updates"); + + AddOutput("ParamOut", "(Tensor) Output parameter"); + AddOutput("AvgSquaredGradOut", + "(Tensor) Output expectation of squared gradient"); + AddOutput("AvgSquaredUpdateOut", + "(Tensor) Output expectation of squared parameter updates"); + + AddAttr("rho", + "(float, default 0.95) Exponential decay rate " + "for squared gradients.") + .SetDefault(0.95f); + AddAttr("epsilon", + "(float, default 1.0e-6) Constant for " + "numerical stability") + .SetDefault(1.0e-6f); + AddComment(R"DOC( +Adadelta Updates Operator. + +This implements the Adadelta optimizer[1]. Adadelta is a per-dimension +adaptive learning rate method for gradient descent. + +Adadelta updates: + +avg_squared_grad_out = rho * avg_squared_grad + (1 - rho) * grad * grad +param_update = - sqrt((avg_squared_update + epsilon) / + (avg_squared_grad_out + epsilon)) * grad +avg_squared_update_out = rho * avg_squared_update + (1 - rho) * param_update**2 +param_out = param + param_update + +References: + [1] ADADELTA: An Adaptive Learning Rate Method + https://arxiv.org/abs/1212.5701 + +)DOC"); + } +}; + +} // namespace operators +} // namespace paddle + +namespace ops = paddle::operators; +REGISTER_OP_WITHOUT_GRADIENT(adadelta, ops::AdadeltaOp, ops::AdadeltaOpMaker); +REGISTER_OP_CPU_KERNEL( + adadelta, ops::AdadeltaOpKernel); diff --git a/paddle/operators/adadelta_op.cu b/paddle/operators/adadelta_op.cu new file mode 100644 index 0000000000..3af1c8c8e9 --- /dev/null +++ b/paddle/operators/adadelta_op.cu @@ -0,0 +1,20 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. */ + +#define EIGEN_USE_GPU +#include "paddle/operators/adadelta_op.h" + +namespace ops = paddle::operators; +REGISTER_OP_GPU_KERNEL( + adadelta, ops::AdadeltaOpKernel); diff --git a/paddle/operators/adadelta_op.h b/paddle/operators/adadelta_op.h new file mode 100644 index 0000000000..d29e15c435 --- /dev/null +++ b/paddle/operators/adadelta_op.h @@ -0,0 +1,69 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once +#include "paddle/framework/eigen.h" +#include "paddle/framework/op_registry.h" + +namespace paddle { +namespace operators { + +template +class AdadeltaOpKernel : public framework::OpKernel { + public: + void Compute(const framework::ExecutionContext& ctx) const override { + auto param_out_tensor = ctx.Output("ParamOut"); + auto avg_squared_grad_out_tensor = + ctx.Output("AvgSquaredGradOut"); + auto avg_squared_update_out_tensor = + ctx.Output("AvgSquaredUpdateOut"); + + param_out_tensor->mutable_data(ctx.GetPlace()); + avg_squared_grad_out_tensor->mutable_data(ctx.GetPlace()); + avg_squared_update_out_tensor->mutable_data(ctx.GetPlace()); + + float rho = ctx.Attr("rho"); + float epsilon = ctx.Attr("epsilon"); + + auto param = framework::EigenVector::Flatten( + *ctx.Input("Param")); + auto grad = framework::EigenVector::Flatten( + *ctx.Input("Grad")); + // Squared gradient accumulator + auto avg_squared_grad = framework::EigenVector::Flatten( + *ctx.Input("AvgSquaredGrad")); + // Squared updates accumulator + auto avg_squared_update = framework::EigenVector::Flatten( + *ctx.Input("AvgSquaredUpdate")); + auto param_out = framework::EigenVector::Flatten(*param_out_tensor); + auto avg_squared_grad_out = + framework::EigenVector::Flatten(*avg_squared_grad_out_tensor); + auto avg_squared_update_out = + framework::EigenVector::Flatten(*avg_squared_update_out_tensor); + auto place = ctx.GetEigenDevice(); + + avg_squared_grad_out.device(place) = + rho * avg_squared_grad + (1 - rho) * grad.square(); + auto update = + -((avg_squared_update + epsilon) / (avg_squared_grad_out + epsilon)) + .sqrt() * + grad; + avg_squared_update_out.device(place) = + rho * avg_squared_update + (1 - rho) * update.square(); + param_out.device(place) = param + update; + } +}; + +} // namespace operators +} // namespace paddle diff --git a/python/paddle/v2/framework/tests/test_adadelta_op.py b/python/paddle/v2/framework/tests/test_adadelta_op.py new file mode 100644 index 0000000000..7105593a98 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_adadelta_op.py @@ -0,0 +1,96 @@ +import unittest +import numpy as np +from op_test import OpTest + + +class TestAdadeltaOp1(OpTest): + def setUp(self): + self.op_type = "adadelta" + param = np.random.uniform(-1, 1, (102, 105)).astype("float32") + grad = np.random.uniform(-1, 1, (102, 105)).astype("float32") + # The squared gradient is positive + avg_squared_grad = np.random.random((102, 105)).astype("float32") + # The squared update is positive + avg_squared_update = np.random.random((102, 105)).astype("float32") + + rho = 0.95 + epsilon = 1e-6 + + self.inputs = { + 'Param': param, + 'Grad': grad, + 'AvgSquaredGrad': avg_squared_grad, + 'AvgSquaredUpdate': avg_squared_update + } + + self.attrs = {'rho': rho, 'epsilon': epsilon} + + avg_squared_grad_out = rho * avg_squared_grad + \ + (1 - rho) * np.square(grad) + update = -np.multiply( + np.sqrt( + np.divide(avg_squared_update + epsilon, avg_squared_grad_out + + epsilon)), grad) + + avg_squared_update_out = rho * avg_squared_update + \ + (1 - rho) * np.square(update) + + param_out = param + update + + self.outputs = { + 'ParamOut': param_out, + 'AvgSquaredGradOut': avg_squared_grad_out, + 'AvgSquaredUpdateOut': avg_squared_update_out + } + + def test_check_output(self): + self.check_output() + + +class TestAdadeltaOp2(OpTest): + '''Test Adadelta op with default attribute values + ''' + + def setUp(self): + self.op_type = "adadelta" + param = np.random.uniform(-1, 1, (102, 105)).astype("float32") + grad = np.random.uniform(-1, 1, (102, 105)).astype("float32") + # The squared gradient is positive + avg_squared_grad = np.random.random((102, 105)).astype("float32") + # The squared update is positive + avg_squared_update = np.random.random((102, 105)).astype("float32") + + rho = 0.95 + epsilon = 1e-6 + + self.inputs = { + 'Param': param, + 'Grad': grad, + 'AvgSquaredGrad': avg_squared_grad, + 'AvgSquaredUpdate': avg_squared_update + } + + avg_squared_grad_out = rho * avg_squared_grad + \ + (1 - rho) * np.square(grad) + update = -np.multiply( + np.sqrt( + np.divide(avg_squared_update + epsilon, avg_squared_grad_out + + epsilon)), grad) + + avg_squared_update_out = rho * avg_squared_update + \ + (1 - rho) * np.square(update) + + param_out = param + update + + self.outputs = { + 'ParamOut': param_out, + 'AvgSquaredGradOut': avg_squared_grad_out, + 'AvgSquaredUpdateOut': avg_squared_update_out + } + + def test_check_output(self): + self.check_output() + + +if __name__ == "__main__": + unittest.main() From 4b07686aa89575442176ce056ef3551a2b31580e Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Thu, 5 Oct 2017 14:42:17 -0700 Subject: [PATCH 153/170] Add unit tests --- paddle/framework/backward.cc | 5 +- paddle/framework/backward_test.cc | 206 ++++++++++++++++++++++++++++++ 2 files changed, 209 insertions(+), 2 deletions(-) diff --git a/paddle/framework/backward.cc b/paddle/framework/backward.cc index efcdd1bc78..c970e01dd1 100644 --- a/paddle/framework/backward.cc +++ b/paddle/framework/backward.cc @@ -289,7 +289,7 @@ std::vector> MakeOpGrad( std::unordered_set& no_grad_vars) { std::vector> grad_op_descs; // All input gradients of forwarding operator do not need to calculat. - const std::vector& inputs = op_desc->InArgumentNames(); + const std::vector& inputs = op_desc->InputArgumentNames(); if (AllGradInSet(inputs, no_grad_vars)) { return grad_op_descs; // empty vector } @@ -323,8 +323,9 @@ std::vector> MakeOpGrad( } } } + for (auto& p : pending_fill_zeros_ops) { - grad_op_descs.push_back(std::move(p)); + grad_op_descs.insert(grad_op_descs.begin(), std::move(p)); } return grad_op_descs; } diff --git a/paddle/framework/backward_test.cc b/paddle/framework/backward_test.cc index 9ea91358f4..30225a4a99 100644 --- a/paddle/framework/backward_test.cc +++ b/paddle/framework/backward_test.cc @@ -155,6 +155,18 @@ class SumOpMaker : public framework::OpProtoAndCheckerMaker { } }; +class MultInOutOpMaker : public OpProtoAndCheckerMaker { + public: + MultInOutOpMaker(OpProto *proto, OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "x"); + AddInput("H", "h"); + AddOutput("Y", "y"); + AddOutput("Z", "z"); + AddComment(""); + } +}; + } // namespace framework } // namespace paddle @@ -172,6 +184,7 @@ REGISTER_OP(sum, f::NOP, f::SumOpMaker, sum_grad, f::NOP); REGISTER_OP_WITHOUT_GRADIENT(fc, f::FcOp, f::FcOpMaker); REGISTER_OP(many_output_op, f::NOP, f::ManyOutputOpMaker, many_output_op_grad, f::NOP); +REGISTER_OP(mult_in_out, f::NOP, f::MultInOutOpMaker, mult_in_out_grad, f::NOP); TEST(Backward, simple_op_not_need_grad) { auto fwd = f::OpRegistry::CreateOp( @@ -487,4 +500,197 @@ TEST(Backward, simple_mult_op) { std::vector({f::GradVarName("out2")})); EXPECT_EQ(grad_op3->Output(f::GradVarName("b")), std::vector({f::GradVarName("b3")})); +} + +TEST(Backward, intermedia_var_no_grad) { + f::ProgramDesc *program_desc = GetNewProgramDesc(); + f::ProgramDescBind &program = f::ProgramDescBind::Instance(program_desc); + f::BlockDescBind *block = program.Block(0); + f::OpDescBind *op1 = block->AppendOp(); + op1->SetType("rowwise_add"); + op1->SetInput("X", {"x1"}); + op1->SetInput("b", {"b1"}); + op1->SetOutput("Out", {"out1"}); + + f::OpDescBind *op2 = block->AppendOp(); + op2->SetType("mul"); + op2->SetInput("X", {"x2"}); + op2->SetInput("Y", {"y2"}); + op2->SetOutput("Out", {"out2"}); + + f::OpDescBind *op3 = block->AppendOp(); + op3->SetType("rowwise_add"); + op3->SetInput("X", {"out2"}); + op3->SetInput("b", {"b3"}); + op3->SetOutput("Out", {"out3"}); + + f::OpDescBind *op4 = block->AppendOp(); + op4->SetType("mul"); + op4->SetInput("X", {"out1"}); + op4->SetInput("Y", {"out3"}); + op4->SetOutput("Out", {"out4"}); + + AppendBackward(program, {"out3"}); + + ASSERT_EQ(block->AllOps().size(), 6UL); + f::OpDescBind *grad_op1 = block->AllOps()[5]; + EXPECT_EQ(grad_op1->Type(), "rowwise_add_grad"); + ASSERT_EQ(grad_op1->InputNames().size(), 1UL); + ASSERT_EQ(grad_op1->OutputNames().size(), 2UL); + EXPECT_EQ(grad_op1->Input(f::GradVarName("Out")), + std::vector({f::GradVarName("out1")})); + EXPECT_EQ(grad_op1->Output(f::GradVarName("X")), + std::vector({f::GradVarName("x1")})); + EXPECT_EQ(grad_op1->Output(f::GradVarName("b")), + std::vector({f::GradVarName("b1")})); + + f::OpDescBind *grad_op4 = block->AllOps()[4]; + EXPECT_EQ(grad_op4->Type(), "mul_grad"); + ASSERT_EQ(grad_op4->InputNames().size(), 4UL); + ASSERT_EQ(grad_op4->OutputNames().size(), 2UL); + EXPECT_EQ(grad_op4->Input("X"), std::vector({"out1"})); + EXPECT_EQ(grad_op4->Input("Y"), std::vector({"out3"})); + EXPECT_EQ(grad_op4->Input("Out"), std::vector({"out4"})); + EXPECT_EQ(grad_op4->Input(f::GradVarName("Out")), + std::vector({f::GradVarName("out4")})); + EXPECT_EQ(grad_op4->Output(f::GradVarName("X")), + std::vector({f::GradVarName("out1")})); + EXPECT_EQ(grad_op4->Output(f::GradVarName("Y")), + std::vector({f::kEmptyVarName})); +} + +TEST(Backward, var_no_grad) { + f::ProgramDesc *program_desc = GetNewProgramDesc(); + f::ProgramDescBind &program = f::ProgramDescBind::Instance(program_desc); + f::BlockDescBind *block = program.Block(0); + f::OpDescBind *op1 = block->AppendOp(); + op1->SetType("mult_in_out"); + op1->SetInput("X", {"x1"}); + op1->SetInput("H", {"h1"}); + op1->SetOutput("Y", {"y1"}); + op1->SetOutput("Z", {"z1"}); + + f::OpDescBind *op2 = block->AppendOp(); + op2->SetType("mult_in_out"); + op2->SetInput("X", {"y1"}); + op2->SetInput("H", {"z1"}); + op2->SetOutput("Y", {"y2"}); + op2->SetOutput("Z", {"z2"}); + + AppendBackward(program, {"z1"}); + + ASSERT_EQ(block->AllOps().size(), 5UL); + f::OpDescBind *grad_op2 = block->AllOps()[2]; + ASSERT_EQ(grad_op2->Type(), "mult_in_out_grad"); + ASSERT_EQ(grad_op2->InputNames().size(), 6UL); + ASSERT_EQ(grad_op2->OutputNames().size(), 2UL); + EXPECT_EQ(grad_op2->Input("X"), std::vector({"y1"})); + EXPECT_EQ(grad_op2->Input("H"), std::vector({"z1"})); + EXPECT_EQ(grad_op2->Input("Y"), std::vector({"y2"})); + EXPECT_EQ(grad_op2->Input("Z"), std::vector({"z2"})); + EXPECT_EQ(grad_op2->Input(f::GradVarName("Y")), + std::vector({f::GradVarName("y2")})); + EXPECT_EQ(grad_op2->Input(f::GradVarName("Z")), + std::vector({f::GradVarName("z2")})); + EXPECT_EQ(grad_op2->Output(f::GradVarName("X")), + std::vector({f::GradVarName("y1")})); + EXPECT_EQ(grad_op2->Output(f::GradVarName("H")), + std::vector({f::kEmptyVarName})); + + f::OpDescBind *fill_zero_op = block->AllOps()[3]; + ASSERT_EQ(fill_zero_op->Type(), "fill_zeros_like"); + ASSERT_EQ(fill_zero_op->InputNames().size(), 1UL); + ASSERT_EQ(fill_zero_op->OutputNames().size(), 1UL); + EXPECT_EQ(fill_zero_op->Input("X"), std::vector({"z1"})); + EXPECT_EQ(fill_zero_op->Output("Y"), + std::vector({std::string("z1") + f::kZeroVarSuffix})); + + f::OpDescBind *grad_op1 = block->AllOps()[4]; + ASSERT_EQ(grad_op1->Type(), "mult_in_out_grad"); + ASSERT_EQ(grad_op1->InputNames().size(), 6UL); + ASSERT_EQ(grad_op1->OutputNames().size(), 2UL); + EXPECT_EQ(grad_op1->Input("X"), std::vector({"x1"})); + EXPECT_EQ(grad_op1->Input("H"), std::vector({"h1"})); + EXPECT_EQ(grad_op1->Input("Y"), std::vector({"y1"})); + EXPECT_EQ(grad_op1->Input("Z"), std::vector({"z1"})); + EXPECT_EQ(grad_op1->Input(f::GradVarName("Y")), + std::vector({f::GradVarName("y1")})); + EXPECT_EQ(grad_op1->Input(f::GradVarName("Z")), + std::vector({std::string("z1") + f::kZeroVarSuffix})); + EXPECT_EQ(grad_op1->Output(f::GradVarName("X")), + std::vector({f::GradVarName("x1")})); + EXPECT_EQ(grad_op1->Output(f::GradVarName("H")), + std::vector({f::GradVarName("h1")})); +} + +TEST(Backward, shared_var) { + f::ProgramDesc *program_desc = GetNewProgramDesc(); + f::ProgramDescBind &program = f::ProgramDescBind::Instance(program_desc); + f::BlockDescBind *block = program.Block(0); + f::OpDescBind *op1 = block->AppendOp(); + op1->SetType("rowwise_add"); + op1->SetInput("X", {"x1"}); + op1->SetInput("b", {"b1"}); + op1->SetOutput("Out", {"out1"}); + + f::OpDescBind *op2 = block->AppendOp(); + op2->SetType("mul"); + op2->SetInput("X", {"out1"}); + op2->SetInput("Y", {"y2"}); + op2->SetOutput("Out", {"out2"}); + + f::OpDescBind *op3 = block->AppendOp(); + op3->SetType("rowwise_add"); + op3->SetInput("X", {"out1"}); + op3->SetInput("b", {"b3"}); + op3->SetOutput("Out", {"out3"}); + + AppendBackward(program, {}); + + ASSERT_EQ(block->AllOps().size(), 7UL); + f::OpDescBind *grad_op3 = block->AllOps()[3]; + ASSERT_EQ(grad_op3->Type(), "rowwise_add_grad"); + ASSERT_EQ(grad_op3->InputNames().size(), 1UL); + ASSERT_EQ(grad_op3->OutputNames().size(), 2UL); + EXPECT_EQ(grad_op3->Input(f::GradVarName("Out")), + std::vector({f::GradVarName("out3")})); + EXPECT_EQ(grad_op3->Output(f::GradVarName("X")), + std::vector({f::GradVarName("out1") + "@RENAME@0"})); + EXPECT_EQ(grad_op3->Output(f::GradVarName("b")), + std::vector({f::GradVarName("b3")})); + + f::OpDescBind *grad_op4 = block->AllOps()[4]; + ASSERT_EQ(grad_op4->Type(), "mul_grad"); + ASSERT_EQ(grad_op4->InputNames().size(), 4UL); + ASSERT_EQ(grad_op4->OutputNames().size(), 2UL); + EXPECT_EQ(grad_op4->Input("X"), std::vector({"out1"})); + EXPECT_EQ(grad_op4->Input("Y"), std::vector({"y2"})); + EXPECT_EQ(grad_op4->Input("Out"), std::vector({"out2"})); + EXPECT_EQ(grad_op4->Input(f::GradVarName("Out")), + std::vector({f::GradVarName("out2")})); + EXPECT_EQ(grad_op4->Output(f::GradVarName("X")), + std::vector({f::GradVarName("out1") + "@RENAME@1"})); + EXPECT_EQ(grad_op4->Output(f::GradVarName("Y")), + std::vector({f::GradVarName("y2")})); + + f::OpDescBind *sum_op = block->AllOps()[5]; + ASSERT_EQ(sum_op->Type(), "sum"); + ASSERT_EQ(sum_op->InputNames().size(), 1UL); + ASSERT_EQ(sum_op->OutputNames().size(), 1UL); + EXPECT_EQ(sum_op->Input("X"), + std::vector({f::GradVarName("out1") + "@RENAME@0", + f::GradVarName("out1") + "@RENAME@1"})); + EXPECT_EQ(sum_op->Output("Out"), + std::vector({f::GradVarName("out1")})); + + f::OpDescBind *grad_op1 = block->AllOps()[6]; + ASSERT_EQ(grad_op1->Type(), "rowwise_add_grad"); + ASSERT_EQ(grad_op1->InputNames().size(), 1UL); + ASSERT_EQ(grad_op1->OutputNames().size(), 2UL); + EXPECT_EQ(grad_op1->Input(f::GradVarName("Out")), + std::vector({f::GradVarName("out1")})); + EXPECT_EQ(grad_op1->Output(f::GradVarName("X")), + std::vector({f::GradVarName("x1")})); + EXPECT_EQ(grad_op1->Output(f::GradVarName("b")), + std::vector({f::GradVarName("b1")})); } \ No newline at end of file From 352af966d7a62a1ed4cedaf9562d05db8026fe23 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Thu, 5 Oct 2017 15:30:03 -0700 Subject: [PATCH 154/170] add python unit test --- paddle/framework/CMakeLists.txt | 4 +- paddle/framework/op_registry.h | 9 +-- paddle/framework/shape_inference.h | 4 ++ paddle/framework/shape_inference_map.cc | 60 ------------------- paddle/framework/shape_inference_map.h | 48 --------------- paddle/pybind/pybind.cc | 24 +++++--- .../v2/framework/tests/test_infer_shape.py | 46 ++++++++++++-- 7 files changed, 62 insertions(+), 133 deletions(-) delete mode 100644 paddle/framework/shape_inference_map.cc delete mode 100644 paddle/framework/shape_inference_map.h diff --git a/paddle/framework/CMakeLists.txt b/paddle/framework/CMakeLists.txt index 986b45451f..a2efcdb55c 100644 --- a/paddle/framework/CMakeLists.txt +++ b/paddle/framework/CMakeLists.txt @@ -26,10 +26,8 @@ cc_library(op_info SRCS op_info.cc DEPS attribute framework_proto proto_desc) cc_library(operator SRCS operator.cc DEPS op_info device_context tensor scope proto_desc) cc_test(operator_test SRCS operator_test.cc DEPS operator op_registry) -cc_library(shape_inference_map SRCS shape_inference_map.cc DEPS op_info operator) - cc_library(grad_op_builder SRCS grad_op_builder.cc DEPS operator proto_desc) -cc_library(op_registry SRCS op_registry.cc DEPS grad_op_builder op_proto_maker op_info shape_inference_map) +cc_library(op_registry SRCS op_registry.cc DEPS grad_op_builder op_proto_maker op_info) cc_test(op_registry_test SRCS op_registry_test.cc DEPS op_registry) cc_test(grad_op_builder_test SRCS grad_op_builder_test.cc DEPS grad_op_builder op_registry sum_op) diff --git a/paddle/framework/op_registry.h b/paddle/framework/op_registry.h index 8138ba117a..ee02da7b4d 100644 --- a/paddle/framework/op_registry.h +++ b/paddle/framework/op_registry.h @@ -26,7 +26,6 @@ limitations under the License. */ #include "paddle/framework/grad_op_builder.h" #include "paddle/framework/operator.h" #include "paddle/framework/scope.h" -#include "paddle/framework/shape_inference_map.h" namespace paddle { namespace framework { @@ -55,16 +54,10 @@ class OpRegistry { const std::string& grad_op_type) { OperatorRegistrar reg(op_type.c_str()); reg.info.grad_op_type_ = grad_op_type; - auto proto = reg.info.Proto(); - std::cout << "====== " << op_type << " =======" << std::endl; - std::cout << proto.SerializeAsString() << std::endl; - std::cout << "=============" << std::endl; - ShapeInferenceMap::Instance().CreateOpWithKernel(reg.info, op_type); + // register gradient op if (!grad_op_type.empty()) { OperatorRegistrar grad_reg(grad_op_type.c_str()); - ShapeInferenceMap::Instance().CreateOpWithKernel(grad_reg.info, - grad_op_type); } } diff --git a/paddle/framework/shape_inference.h b/paddle/framework/shape_inference.h index ac6f238638..8189823c19 100644 --- a/paddle/framework/shape_inference.h +++ b/paddle/framework/shape_inference.h @@ -20,6 +20,10 @@ limitations under the License. */ namespace paddle { namespace framework { +class InferShapeContextBase; + +typedef std::function InferShapeFn; + class InferShapeContextBase { public: virtual ~InferShapeContextBase() {} diff --git a/paddle/framework/shape_inference_map.cc b/paddle/framework/shape_inference_map.cc deleted file mode 100644 index bd2b867984..0000000000 --- a/paddle/framework/shape_inference_map.cc +++ /dev/null @@ -1,60 +0,0 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ - -#include "paddle/framework/shape_inference_map.h" - -namespace paddle { -namespace framework { - -static VariableNameMap ConvertOpProtoVarsToVarNameMap( - const google::protobuf::RepeatedPtrField& op_proto_vars) { - VariableNameMap ret_val; - for (auto& var : op_proto_vars) { - ret_val[var.name()] = {}; - } - return ret_val; -} - -static ShapeInferenceMap* g_shape_inference_map = nullptr; - -ShapeInferenceMap& ShapeInferenceMap::Instance() { - if (g_shape_inference_map == nullptr) { - g_shape_inference_map = new ShapeInferenceMap(); - } - return *g_shape_inference_map; -} - -void ShapeInferenceMap::CreateOpWithKernel(const OpInfo& op_info, - const std::string& op_type) { - auto proto = op_info.Proto(); - std::cout << "========= " << op_type << " in======" << std::endl; - std::cout << proto.SerializeAsString() << std::endl; - std::cout << "========= " << op_type << " out======" << std::endl; - const VariableNameMap inputs = ConvertOpProtoVarsToVarNameMap(proto.inputs()); - const VariableNameMap outputs = - ConvertOpProtoVarsToVarNameMap(proto.outputs()); - auto* op = op_info.Creator()(op_type, inputs, outputs, {}); - auto* op_with_kernel = dynamic_cast(op); - auto it = op_shape_inference_map_.find(op_type); - if (it != op_shape_inference_map_.end()) { - PADDLE_THROW("OpWithKernel(%s) is already registered for infer_shape", - op_type); - } - if (op_with_kernel != nullptr) { - op_shape_inference_map_[op_type] = op_with_kernel; - } -} - -} // namespace framework -} // namespace paddle diff --git a/paddle/framework/shape_inference_map.h b/paddle/framework/shape_inference_map.h deleted file mode 100644 index 6c7304f6c0..0000000000 --- a/paddle/framework/shape_inference_map.h +++ /dev/null @@ -1,48 +0,0 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ - -#pragma once - -#include - -#include "paddle/framework/op_info.h" -#include "paddle/framework/operator.h" -#include "paddle/framework/shape_inference.h" - -namespace paddle { -namespace framework { - -class ShapeInferenceMap { - public: - static ShapeInferenceMap& Instance(); - - void CreateOpWithKernel(const OpInfo& op_info, const std::string& op_type); - - OperatorWithKernel* GetOpWithKernel(const std::string& op_type) { - auto it = op_shape_inference_map_.find(op_type); - if (it == op_shape_inference_map_.end()) { - return nullptr; - } - return it->second; - } - - private: - ShapeInferenceMap() = default; - DISABLE_COPY_AND_ASSIGN(ShapeInferenceMap); - - std::unordered_map op_shape_inference_map_; -}; - -} // namespace framework -} // namespace paddle diff --git a/paddle/pybind/pybind.cc b/paddle/pybind/pybind.cc index e11bcc0e0f..2ad0344c09 100644 --- a/paddle/pybind/pybind.cc +++ b/paddle/pybind/pybind.cc @@ -223,15 +223,21 @@ All parameter, weight, gradient are variables in Paddle. desc.InitializationErrorString()); return OpRegistry::CreateOp(desc); }) - .def("infer_shape", - [](const OpDescBind &op_desc, BlockDescBind &block) { - auto &shape_inference_map = ShapeInferenceMap::Instance(); - auto *op = shape_inference_map.GetOpWithKernel(op_desc.Type()); - if (op != nullptr) { - auto ctx = CompileTimeInferShapeContext(op_desc, block); - op->InferShape(&ctx); - } - }) + .def_static("infer_shape", + [](OpDescBind &op_desc, BlockDescBind &block) { + auto op = OpRegistry::CreateOp(*op_desc.Proto()); + auto *op_with_kernel = + dynamic_cast(op.get()); + if (op_with_kernel != nullptr) { + auto ctx = CompileTimeInferShapeContext(op_desc, block); + op_with_kernel->InferShape(&ctx); + } else { + PADDLE_THROW( + "OP(%s) is not type of OperatorWithKernel, " + "should not call this function", + op_desc.Type()); + } + }) .def("backward", [](const OperatorBase &forwardOp, const std::unordered_set &no_grad_vars) { diff --git a/python/paddle/v2/framework/tests/test_infer_shape.py b/python/paddle/v2/framework/tests/test_infer_shape.py index 56d3a90123..ec93aaf843 100644 --- a/python/paddle/v2/framework/tests/test_infer_shape.py +++ b/python/paddle/v2/framework/tests/test_infer_shape.py @@ -10,11 +10,13 @@ class TestInferShape(unittest.TestCase): block = prog.block(0) self.assertIsNotNone(block) + shape = [10, 20] + # prepare input/output x1 = block.new_var("x1") - x1.set_shape([10, 20]) + x1.set_shape(shape) x2 = block.new_var("x2") - x2.set_shape([10, 20]) + x2.set_shape(shape) out = block.new_var("out") @@ -24,6 +26,40 @@ class TestInferShape(unittest.TestCase): sum_op_desc.set_input("X", ["x1", "x2"]) sum_op_desc.set_output("Out", ["out"]) - sum_op = Operator("sum", X=["x1", "x2"], Out="out") - sum_op.infer_shape(sum_op_desc, block) - print(out.shape()) + print(type(sum_op_desc)) + print(type(block)) + core.Operator.infer_shape(sum_op_desc, block) + self.assertEqual(out.shape(), shape) + + def test_mul_op(self): + prog = core.ProgramDesc.__create_program_desc__() + self.assertIsNotNone(prog) + block = prog.block(0) + self.assertIsNotNone(block) + + x_shape = [10, 20] + y_shape = [20, 30] + + # prepare input/output + x1 = block.new_var("x") + x1.set_shape(x_shape) + x2 = block.new_var("y") + x2.set_shape(y_shape) + + out = block.new_var("out") + + # prepare the operator + mul_op_desc = block.append_op() + mul_op_desc.set_type("mul") + mul_op_desc.set_input("X", ["x"]) + mul_op_desc.set_input("Y", ["y"]) + mul_op_desc.set_output("Out", ["out"]) + mul_op_desc.set_attr("x_num_col_dims", 1) + mul_op_desc.set_attr("y_num_col_dims", 1) + + core.Operator.infer_shape(mul_op_desc, block) + self.assertEqual(out.shape(), [x_shape[0], y_shape[1]]) + + +if __name__ == '__main__': + unittest.main() From 628715d602491732f78f314d355760ca8ba326e3 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Thu, 5 Oct 2017 15:43:51 -0700 Subject: [PATCH 155/170] clean code --- paddle/framework/shape_inference.h | 5 ----- python/paddle/v2/framework/tests/test_infer_shape.py | 2 -- 2 files changed, 7 deletions(-) diff --git a/paddle/framework/shape_inference.h b/paddle/framework/shape_inference.h index 8189823c19..bc8af0eb3e 100644 --- a/paddle/framework/shape_inference.h +++ b/paddle/framework/shape_inference.h @@ -14,16 +14,11 @@ limitations under the License. */ #pragma once -#include "paddle/framework/attribute.h" #include "paddle/framework/ddim.h" namespace paddle { namespace framework { -class InferShapeContextBase; - -typedef std::function InferShapeFn; - class InferShapeContextBase { public: virtual ~InferShapeContextBase() {} diff --git a/python/paddle/v2/framework/tests/test_infer_shape.py b/python/paddle/v2/framework/tests/test_infer_shape.py index ec93aaf843..b38ec9c037 100644 --- a/python/paddle/v2/framework/tests/test_infer_shape.py +++ b/python/paddle/v2/framework/tests/test_infer_shape.py @@ -26,8 +26,6 @@ class TestInferShape(unittest.TestCase): sum_op_desc.set_input("X", ["x1", "x2"]) sum_op_desc.set_output("Out", ["out"]) - print(type(sum_op_desc)) - print(type(block)) core.Operator.infer_shape(sum_op_desc, block) self.assertEqual(out.shape(), shape) From e043b386606afc95ffd9135c14866d5e3b77b642 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Thu, 5 Oct 2017 15:47:42 -0700 Subject: [PATCH 156/170] clean code --- paddle/framework/op_registry.h | 1 - 1 file changed, 1 deletion(-) diff --git a/paddle/framework/op_registry.h b/paddle/framework/op_registry.h index ee02da7b4d..4ee2c7d275 100644 --- a/paddle/framework/op_registry.h +++ b/paddle/framework/op_registry.h @@ -54,7 +54,6 @@ class OpRegistry { const std::string& grad_op_type) { OperatorRegistrar reg(op_type.c_str()); reg.info.grad_op_type_ = grad_op_type; - // register gradient op if (!grad_op_type.empty()) { OperatorRegistrar grad_reg(grad_op_type.c_str()); From 154a6ed29c13eeab9c4f785cf5ca1520ba8ca999 Mon Sep 17 00:00:00 2001 From: Kavya Srinet Date: Thu, 5 Oct 2017 17:52:10 -0700 Subject: [PATCH 157/170] Implementing tanhshrink operator --- paddle/operators/activation_op.cc | 14 +++++++++++++ paddle/operators/activation_op.h | 21 ++++++++++++++++++- .../v2/framework/tests/test_activation_op.py | 15 +++++++++++++ 3 files changed, 49 insertions(+), 1 deletion(-) diff --git a/paddle/operators/activation_op.cc b/paddle/operators/activation_op.cc index 5f2ecc2673..66e9d2c401 100644 --- a/paddle/operators/activation_op.cc +++ b/paddle/operators/activation_op.cc @@ -97,6 +97,17 @@ class TanhOpMaker : public framework::OpProtoAndCheckerMaker { } }; +class TanhShrinkOpMaker : public framework::OpProtoAndCheckerMaker { + public: + TanhShrinkOpMaker(framework::OpProto *proto, + framework::OpAttrChecker *op_checker) + : OpProtoAndCheckerMaker(proto, op_checker) { + AddInput("X", "Input of TanhShrink operator"); + AddOutput("Y", "Output of TanhShrink operator"); + AddComment("TanhShrink activation operator, tanhshrink(x) = x - tanh(x)"); + } +}; + class SqrtOpMaker : public framework::OpProtoAndCheckerMaker { public: SqrtOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) @@ -235,6 +246,9 @@ REGISTER_OP(relu, ops::ActivationOp, ops::ReluOpMaker, relu_grad, REGISTER_OP(tanh, ops::ActivationOp, ops::TanhOpMaker, tanh_grad, ops::ActivationOpGrad); +REGISTER_OP(tanh_shrink, ops::ActivationOp, ops::TanhShrinkOpMaker, + tanh_shrink_grad, ops::ActivationOpGrad); + REGISTER_OP(sqrt, ops::ActivationOp, ops::SqrtOpMaker, sqrt_grad, ops::ActivationOpGrad); diff --git a/paddle/operators/activation_op.h b/paddle/operators/activation_op.h index dae66cc77d..2450601742 100644 --- a/paddle/operators/activation_op.h +++ b/paddle/operators/activation_op.h @@ -146,6 +146,24 @@ struct TanhGradFunctor : public BaseActivationFunctor { } }; +// tanhshrink(x) = x - tanh(x) +// where tanh(x) = (exp(x) - exp(-x)) / (exp(x) + exp(-x)) +template +struct TanhShrinkFunctor : public BaseActivationFunctor { + template + void operator()(Device d, X x, Y y) const { + y.device(d) = x - x.tanh(); + } +}; + +template +struct TanhShrinkGradFunctor : public BaseActivationFunctor { + template + void operator()(Device d, X x, Y y, dY dy, dX dx) const { + dx.device(d) = dy * (x.tanh() * x.tanh()); + } +}; + // sqrt(x) = x^(1/2) template struct SqrtFunctor : public BaseActivationFunctor { @@ -407,4 +425,5 @@ struct STanhGradFunctor : public BaseActivationFunctor { __macro(pow, PowFunctor, PowGradFunctor); \ __macro(stanh, STanhFunctor, STanhGradFunctor); \ __macro(softsign, SoftsignFunctor, SoftsignGradFunctor); \ - __macro(leaky_relu, LeakyReluFunctor, LeakyReluGradFunctor) + __macro(leaky_relu, LeakyReluFunctor, LeakyReluGradFunctor); \ + __macro(tanh_shrink, TanhShrinkFunctor, TanhShrinkGradFunctor) diff --git a/python/paddle/v2/framework/tests/test_activation_op.py b/python/paddle/v2/framework/tests/test_activation_op.py index f232996a55..701e1a1aee 100644 --- a/python/paddle/v2/framework/tests/test_activation_op.py +++ b/python/paddle/v2/framework/tests/test_activation_op.py @@ -48,6 +48,21 @@ class TestTanh(OpTest): self.check_grad(['X'], 'Y', max_relative_error=0.007) +class TestTanhShrink(OpTest): + def setUp(self): + self.op_type = "tanh_shrink" + self.inputs = { + 'X': np.random.uniform(0.1, 1, [10, 17]).astype("float32") + } + self.outputs = {'Y': self.inputs['X'] - np.tanh(self.inputs['X'])} + + def test_check_output(self): + self.check_output() + + def test_check_grad(self): + self.check_grad(['X'], 'Y', max_relative_error=0.008) + + class TestSqrt(OpTest): def setUp(self): self.op_type = "sqrt" From f52cdaa0cee682ddc3588286af42d960141596f0 Mon Sep 17 00:00:00 2001 From: Kavya Srinet Date: Thu, 5 Oct 2017 19:12:27 -0700 Subject: [PATCH 158/170] Updated RMSProp to have learning rate as an input and work with GPU --- paddle/operators/rmsprop_op.h | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/paddle/operators/rmsprop_op.h b/paddle/operators/rmsprop_op.h index 9c04276ec6..7bf2129010 100644 --- a/paddle/operators/rmsprop_op.h +++ b/paddle/operators/rmsprop_op.h @@ -32,6 +32,8 @@ class RmspropOpKernel : public framework::OpKernel { auto* moment_out = ctx.Output("MomentOut"); auto* mean_square_out = ctx.Output("MeanSquareOut"); + auto grad = ctx.Input("Grad"); + param_out->mutable_data(ctx.GetPlace()); moment_out->mutable_data(ctx.GetPlace()); mean_square_out->mutable_data(ctx.GetPlace()); @@ -42,8 +44,8 @@ class RmspropOpKernel : public framework::OpKernel { auto p = EigenVector::Flatten(*ctx.Input("Param")); auto ms = EigenVector::Flatten(*ctx.Input("MeanSquare")); - float lr = ctx.Input("LearningRate")->data()[0]; - auto g = EigenVector::Flatten(*ctx.Input("Grad")); + auto lr = EigenVector::Flatten(*ctx.Input("LearningRate")); + auto g = EigenVector::Flatten(*grad); auto mom = EigenVector::Flatten(*ctx.Input("Moment")); auto p_out = EigenVector::Flatten(*param_out); @@ -51,8 +53,12 @@ class RmspropOpKernel : public framework::OpKernel { auto ms_out = EigenVector::Flatten(*mean_square_out); auto place = ctx.GetEigenDevice(); + Eigen::DSizes grad_dsize(grad->numel()); + ms_out.device(place) = rho * ms + (1 - rho) * g * g; - mom_out.device(place) = momentum * mom + lr * g / (ms_out + epsilon).sqrt(); + mom_out.device(place) = + momentum * mom + + lr.broadcast(grad_dsize) * g / (ms_out + epsilon).sqrt(); p_out.device(place) = p - mom_out; } }; From bddaef6218b7e32bb510d348396d81c96ba29a3a Mon Sep 17 00:00:00 2001 From: Abhinav Arora Date: Thu, 5 Oct 2017 21:03:49 -0700 Subject: [PATCH 159/170] Correcting grammatical mistakes in the python_api doc --- doc/design/python_api.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/design/python_api.md b/doc/design/python_api.md index 5c68354274..6213da65c8 100644 --- a/doc/design/python_api.md +++ b/doc/design/python_api.md @@ -15,9 +15,9 @@ Please be aware that these Python classes need to maintain some construction-tim ### Program -A `ProgramDesc` describes a [DL program](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/program.md), which is composed of an array of `BlockDesc`s. A `BlockDesc` refers to its parent block by its index in the array. For example, operators in the step block of an RNN operator needs to be able to access variables in its ancessor blocks. +A `ProgramDesc` describes a [DL program](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/program.md), which is composed of an array of `BlockDesc`s. The `BlockDesc`s in a `ProgramDesc` can have a tree-like hierarchical structure. However, the `ProgramDesc` onlys stores a flattened array of `BlockDesc`s. A `BlockDesc` refers to its parent block by its index in the array. For example, operators in the step block of an RNN operator need to be able to access variables in its ancestor blocks. -Whenever we create a block, we need set its parent block to the current block, so the Python class `Program` needs to maintain a data member `current_block`. +Whenever we create a block, we need to set its parent block to the current block, hence the Python class `Program` needs to maintain a data member `current_block`. ```python class Program(objects): @@ -81,13 +81,13 @@ class Block(objects): self.ops.prepend(Operator(self, ...)) ``` -`create_parameter` is necessary because parameters are global variables, those defined in the global block, but can be created in some sub-blocks, e.g., an FC layer in the step block of an RNN operator. +`create_parameter` is necessary because parameters are global variables, defined in the global block, but can be created in some sub-blocks. For example, an FC layer in the step block of an RNN operator. -`prepand_operator` is necessary because the constructor of `Parameter` needs to create the initialize (or load) operator of the parameter, and would like to put it in the *preamble* of the global block. +`prepend_operator` is necessary because the constructor of `Parameter` needs to create the initialize (or load) operator of the parameter, and would like to put it in the *preamble* of the global block. ### Operator -The `Operator` class fills in the `OpDesc` message and calls the C++ function `InferShape` to infer output shape from input shape. +The `Operator` class fills in the `OpDesc` message and calls the C++ function `InferShape` to infer the output shapes from the input shapes. ```python class Operator(object): @@ -105,7 +105,7 @@ class Operator(object): return self.proto.type() ``` -`Operator` creates the `OpDesc` message in C++ space, so could it call the `InferShape` function, which is in C++. +`Operator` creates the `OpDesc` message in C++ space, so that it can call the `InferShape` function, which is in C++. ### Variable @@ -128,7 +128,7 @@ class Variable(object): self.writer = None ``` -Please be aware of `self.writer`, that tracks operator who creates the variable. It possible that there are more than one operators who write a variable, but in Python space, each writes to a variable is represented by a Variable class. This is guaranteed by the fact that **`core.NewVarDesc` must NOT create a new `VarDesc` message if its name already exists in the specified block**. +Please be aware of `self.writer`, that tracks operator who creates the variable. It possible that there are more than one operators who write a variable, but in Python space, each write to a variable is represented by a Variable class. This is guaranteed by the fact that **`core.NewVarDesc` must NOT create a new `VarDesc` message if its name already exists in the specified block**. ### Parameter @@ -155,7 +155,7 @@ class Parameter(Variable): initialize_op_attrs) ``` -When users create a parameter, s/he can call +When users create a parameter, they can call ```python program.create_parameter( From 78f4c803f370f461f9e62e905b7870a65d05b55c Mon Sep 17 00:00:00 2001 From: Kexin Zhao Date: Thu, 5 Oct 2017 21:47:25 -0700 Subject: [PATCH 160/170] change learning rate and fix format --- paddle/operators/adagrad_op.cc | 67 ++++++++++--------- paddle/operators/adagrad_op.h | 43 ++++++------ .../v2/framework/tests/test_adagrad_op.py | 48 +++++++++++-- 3 files changed, 97 insertions(+), 61 deletions(-) diff --git a/paddle/operators/adagrad_op.cc b/paddle/operators/adagrad_op.cc index 56a5fbcb86..ea2ff3c503 100644 --- a/paddle/operators/adagrad_op.cc +++ b/paddle/operators/adagrad_op.cc @@ -23,33 +23,33 @@ class AdagradOp : public framework::OperatorWithKernel { protected: void InferShape(framework::InferShapeContextBase *ctx) const override { - PADDLE_ENFORCE(ctx->HasInput("param"), - "Input(param) of AdagradOp should not be null."); - PADDLE_ENFORCE(ctx->HasInput("grad"), - "Input(grad) of AdagradOp should not be null."); - PADDLE_ENFORCE(ctx->HasInput("moment"), - "Input(moment) of AdagradOp should not be null."); - PADDLE_ENFORCE(ctx->HasInput("learning_rate"), - "Input(learning_rate) of AdagradOp should not be null."); - - PADDLE_ENFORCE(ctx->HasOutput("param_out"), - "Output(param_out) of AdagradOp should not be null."); - PADDLE_ENFORCE(ctx->HasOutput("moment_out"), - "Output(moment_out) of AdagradOp should not be null."); - - auto lr_dims = ctx->GetInputDim("learning_rate"); + PADDLE_ENFORCE(ctx->HasInput("Param"), + "Input(Param) of AdagradOp should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Grad"), + "Input(Grad) of AdagradOp should not be null."); + PADDLE_ENFORCE(ctx->HasInput("Moment"), + "Input(Moment) of AdagradOp should not be null."); + PADDLE_ENFORCE(ctx->HasInput("LearningRate"), + "Input(LearningRate) of AdagradOp should not be null."); + + PADDLE_ENFORCE(ctx->HasOutput("ParamOut"), + "Output(ParamOut) of AdagradOp should not be null."); + PADDLE_ENFORCE(ctx->HasOutput("MomentOut"), + "Output(MomentOut) of AdagradOp should not be null."); + + auto lr_dims = ctx->GetInputDim("LearningRate"); PADDLE_ENFORCE_EQ(framework::product(lr_dims), 1, - "learning_rate should have one element"); - auto param_dim = ctx->GetInputDim("param"); + "LearningRate should have one element"); + auto param_dims = ctx->GetInputDim("Param"); PADDLE_ENFORCE_EQ( - param_dim, ctx->GetInputDim("grad"), - "Param and grad input of AdagradOp should have the same dimension."); + param_dims, ctx->GetInputDim("Grad"), + "Param and Grad input of AdagradOp should have the same dimension."); PADDLE_ENFORCE_EQ( - param_dim, ctx->GetInputDim("moment"), - "Param and moment input of AdagradOp should have the same dimension."); + param_dims, ctx->GetInputDim("Moment"), + "Param and Moment input of AdagradOp should have the same dimension."); - ctx->SetOutputDim("param_out", param_dim); - ctx->SetOutputDim("moment_out", param_dim); + ctx->SetOutputDim("ParamOut", param_dims); + ctx->SetOutputDim("MomentOut", param_dims); } }; @@ -58,15 +58,18 @@ class AdagradOpMaker : public framework::OpProtoAndCheckerMaker { AdagradOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker) : OpProtoAndCheckerMaker(proto, op_checker) { - AddInput("param", "Input parameter"); - AddInput("grad", "Input gradient"); - AddInput("moment", "Second moment"); - AddInput("learning_rate", "learning rate of adagrad"); - - AddOutput("param_out", "Output parameter"); - AddOutput("moment_out", "Output second moment"); - - AddAttr("epsilon", "Constant for numerical stability"); + AddInput("Param", "(Tensor) Input parameter"); + AddInput("Grad", "(Tensor) Input gradient"); + AddInput("Moment", "(Tensor) Second moment"); + AddInput("LearningRate", "(Tensor) Learning rate"); + + AddOutput("ParamOut", "(Tensor) Output parameter"); + AddOutput("MomentOut", "(Tensor) Output second moment"); + + AddAttr("epsilon", + "(float, default 1.0e-6) " + "Constant for numerical stability") + .SetDefault(1.0e-6f); AddComment(R"DOC( Adaptive Gradient Algorithm (Adagrad). diff --git a/paddle/operators/adagrad_op.h b/paddle/operators/adagrad_op.h index 73833d4a3f..c5d8f751d3 100644 --- a/paddle/operators/adagrad_op.h +++ b/paddle/operators/adagrad_op.h @@ -19,40 +19,35 @@ limitations under the License. */ namespace paddle { namespace operators { -using Tensor = framework::Tensor; - -template -using EigenScalar = framework::EigenScalar; - -template -using EigenVector = framework::EigenVector; - template class AdagradOpKernel : public framework::OpKernel { public: void Compute(const framework::ExecutionContext& ctx) const override { - auto param_out = ctx.Output("param_out"); - auto moment_out = ctx.Output("moment_out"); + auto param_out_tensor = ctx.Output("ParamOut"); + auto moment_out_tensor = ctx.Output("MomentOut"); - param_out->mutable_data(ctx.GetPlace()); - moment_out->mutable_data(ctx.GetPlace()); + param_out_tensor->mutable_data(ctx.GetPlace()); + moment_out_tensor->mutable_data(ctx.GetPlace()); - float lr = ctx.Input("learning_rate")->data()[0]; float epsilon = ctx.Attr("epsilon"); - auto p = EigenVector::Flatten(*ctx.Input("param")); - auto g = EigenVector::Flatten(*ctx.Input("grad")); - auto m = EigenVector::Flatten(*ctx.Input("moment")); - auto lr = EigenScalar::From(*ctx.Input("learning_rate")); - - auto p_out = EigenVector::Flatten(*param_out); - auto m_out = EigenVector::Flatten(*moment_out); + auto param = framework::EigenVector::Flatten( + *ctx.Input("Param")); + auto grad = framework::EigenVector::Flatten( + *ctx.Input("Grad")); + auto moment = framework::EigenVector::Flatten( + *ctx.Input("Moment")); + auto lr = framework::EigenVector::Flatten( + *ctx.Input("LearningRate")); + + auto param_out = framework::EigenVector::Flatten(*param_out_tensor); + auto moment_out = framework::EigenVector::Flatten(*moment_out_tensor); auto place = ctx.GetEigenDevice(); - m_out.device(place) = m + g * g; - p_out.device(place) = p - lr * g / (m_out.sqrt() + epsilon); + moment_out.device(place) = moment + grad * grad; + Eigen::DSizes m_dsize(moment_out_tensor->numel()); + param_out.device(place) = + param - lr.broadcast(m_dsize) * grad / (moment_out.sqrt() + epsilon); } }; diff --git a/python/paddle/v2/framework/tests/test_adagrad_op.py b/python/paddle/v2/framework/tests/test_adagrad_op.py index 2ee38ea37c..66bad349e5 100644 --- a/python/paddle/v2/framework/tests/test_adagrad_op.py +++ b/python/paddle/v2/framework/tests/test_adagrad_op.py @@ -3,25 +3,63 @@ import numpy as np from op_test import OpTest -class TestAdagradOp(OpTest): +class TestAdagradOp1(OpTest): + ''' Test Adagrad operator with explicit attributes + ''' + def setUp(self): self.op_type = "adagrad" param = np.random.random((123, 321)).astype("float32") grad = np.random.random((123, 321)).astype("float32") moment = np.zeros((123, 321)).astype("float32") + lr = 0.01 + epsilon = 1e-8 + + self.inputs = { + 'Param': param, + 'Grad': grad, + 'Moment': moment, + 'LearningRate': np.array([lr]).astype("float32") + } + + self.attrs = {'epsilon': epsilon} + + moment_out = moment + grad * grad + param_out = param - lr * grad / (np.sqrt(moment_out) + epsilon) + + self.outputs = {'ParamOut': param_out, 'MomentOut': moment_out} + + def test_check_output(self): + self.check_output() + + +class TestAdagradOp2(OpTest): + ''' Test Adagrad operator with default attributes + ''' - lr = np.array([0.01]).astype("float32") + def setUp(self): + self.op_type = "adagrad" + + param = np.random.random((123, 321)).astype("float32") + grad = np.random.random((123, 321)).astype("float32") + moment = np.zeros((123, 321)).astype("float32") + lr = 0.01 epsilon = 1e-6 - self.inputs = {'param': param, 'grad': grad, 'moment': moment} + self.inputs = { + 'Param': param, + 'Grad': grad, + 'Moment': moment, + 'LearningRate': np.array([lr]).astype("float32") + } - self.attrs = {'learning_rate': learning_rate, 'epsilon': epsilon} + self.attrs = {'epsilon': epsilon} moment_out = moment + grad * grad param_out = param - lr * grad / (np.sqrt(moment_out) + epsilon) - self.outputs = {'param_out': param_out, 'moment_out': moment_out} + self.outputs = {'ParamOut': param_out, 'MomentOut': moment_out} def test_check_output(self): self.check_output() From 8c7ee7a5cdc6e959ad55a6a24d3ffe4bc3f37144 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Fri, 6 Oct 2017 11:41:59 -0700 Subject: [PATCH 161/170] optimize code --- paddle/framework/block_desc.cc | 3 +-- paddle/framework/shape_inference.h | 3 +++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/paddle/framework/block_desc.cc b/paddle/framework/block_desc.cc index 670533a3fe..01f50e1393 100644 --- a/paddle/framework/block_desc.cc +++ b/paddle/framework/block_desc.cc @@ -35,8 +35,7 @@ VarDescBind *BlockDescBind::Var(const std::string &name) const { } bool BlockDescBind::HasVar(const std::string &name) const { - auto it = vars_.find(name); - return it != vars_.end(); + return vars_.find(name) != vars_.end(); } std::vector BlockDescBind::AllVars() const { diff --git a/paddle/framework/shape_inference.h b/paddle/framework/shape_inference.h index bc8af0eb3e..74e0371e32 100644 --- a/paddle/framework/shape_inference.h +++ b/paddle/framework/shape_inference.h @@ -19,6 +19,9 @@ limitations under the License. */ namespace paddle { namespace framework { +// TODO(longfei): Once after both CompileTimeInferShapeContext and +// RuntimeInferShapeContext get merged, we can rename InferShapeContextBase into +// InferShapeContext so to replace the current InferShapeContext. class InferShapeContextBase { public: virtual ~InferShapeContextBase() {} From 3e82922851da5f36f8e8187b818a1f43f324c64f Mon Sep 17 00:00:00 2001 From: Yan Chunwei Date: Fri, 6 Oct 2017 15:53:54 -0400 Subject: [PATCH 162/170] fix tensorarray unpack bug (#4614) --- paddle/framework/tensor_array.cc | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/paddle/framework/tensor_array.cc b/paddle/framework/tensor_array.cc index d54714c66c..2728bce1c1 100644 --- a/paddle/framework/tensor_array.cc +++ b/paddle/framework/tensor_array.cc @@ -217,12 +217,11 @@ LoDTensor DynamicBatchUnpacker::GetBatch(size_t index) { // collect indice need to copy to the batch std::vector indice; - for (size_t seq_id = 0; seq_id < meta.size(); seq_id++) { - const auto& seq_meta = meta[seq_id]; - if (index >= seq_meta.end) break; - indice.push_back(seq_meta.begin + index); + for (const auto& seq : meta) { + size_t id = seq.begin + index; + if (id >= seq.end) break; + indice.push_back(id); } - PADDLE_ENFORCE(!indice.empty(), "invalid batch at %d", index); // copy the indice of records in LoDTensor @@ -232,16 +231,18 @@ LoDTensor DynamicBatchUnpacker::GetBatch(size_t index) { result.Resize(make_ddim(record_dims_vec)); result.mutable_data(platform::CPUPlace()); - for (size_t i = 0; i < indice.size() - 1; i++) { + for (size_t i = 0; i < indice.size(); i++) { auto index = indice[i]; auto target = result.Slice(i, i + 1); auto source_ = source->Slice(index, index + 1); + target.CopyFrom(source_, platform::CPUPlace()); } return result; } +// TODO(supejom) to cache lod if reasonable LoDTensor PackDynamicBatch(const std::vector& source, const std::vector& meta, const LoD& lod, size_t level) { @@ -273,7 +274,6 @@ LoDTensor PackDynamicBatch(const std::vector& source, } result.set_lod(lod); - return result; } From 4acd5abaaa6c18109e505bf2b2ef07a03df25ccc Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Fri, 6 Oct 2017 13:25:56 -0700 Subject: [PATCH 163/170] update comment for input/output length check --- paddle/framework/operator.h | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/paddle/framework/operator.h b/paddle/framework/operator.h index 458404af6d..d7bc9c9ffb 100644 --- a/paddle/framework/operator.h +++ b/paddle/framework/operator.h @@ -325,15 +325,21 @@ class CompileTimeInferShapeContext : public InferShapeContextBase { bool HasInput(const std::string& name) const override { const std::vector& input_names = op_.Input(name); - PADDLE_ENFORCE_EQ(input_names.size(), 1UL, "Inputs(%s) length is not 1", - name); + auto length = input_names.size(); + PADDLE_ENFORCE_EQ(length, 1UL, + "Input(%s) should have only one value, " + "but it have %d now", + name, length); return block_.HasVar(input_names[0]); } bool HasOutput(const std::string& name) const override { const std::vector& output_names = op_.Output(name); - PADDLE_ENFORCE_EQ(output_names.size(), 1UL, "Outputs(%s) length is not 1", - name); + auto length = output_names.size(); + PADDLE_ENFORCE_EQ(length, 1UL, + "Output(%s) should have only one value, " + "but it have %d now", + name, length); return block_.HasVar(output_names[0]); } @@ -357,7 +363,11 @@ class CompileTimeInferShapeContext : public InferShapeContextBase { DDim GetInputDim(const std::string& name) const override { std::vector ddims = GetInputsDim(name); - PADDLE_ENFORCE_EQ(ddims.size(), 1UL, "Inputs(%s) length is not 1", name); + auto length = ddims.size(); + PADDLE_ENFORCE_EQ(length, 1UL, + "Input(%s) should have 1 value, " + "but it has %d now", + name, length); return ddims[0]; } @@ -367,7 +377,11 @@ class CompileTimeInferShapeContext : public InferShapeContextBase { DDim GetOutputDim(const std::string& name) const override { std::vector ddims = GetOutputsDim(name); - PADDLE_ENFORCE_EQ(ddims.size(), 1UL, "Outputs(%s) length is not 1", name); + auto length = ddims.size(); + PADDLE_ENFORCE_EQ(length, 1UL, + "Output(%s) should have 1 value, " + "but it has %d now", + name, length); return ddims[0]; } From 803b7b6298bea773316b3030b2edbc2fa183b738 Mon Sep 17 00:00:00 2001 From: fengjiayi Date: Fri, 6 Oct 2017 16:01:14 -0700 Subject: [PATCH 164/170] Follow comments --- paddle/framework/op_info.h | 2 ++ paddle/framework/op_registry.cc | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/paddle/framework/op_info.h b/paddle/framework/op_info.h index 231f212fa3..c504f69e30 100644 --- a/paddle/framework/op_info.h +++ b/paddle/framework/op_info.h @@ -54,6 +54,8 @@ struct OpInfo { "Operator GradOpMaker has not been registered."); return grad_op_maker_; } + + const OpAttrChecker* Checker() const { return checker_; } }; class OpInfoMap { diff --git a/paddle/framework/op_registry.cc b/paddle/framework/op_registry.cc index e9d2e55872..504afbd5db 100644 --- a/paddle/framework/op_registry.cc +++ b/paddle/framework/op_registry.cc @@ -23,8 +23,8 @@ std::unique_ptr OpRegistry::CreateOp( const std::string& type, const VariableNameMap& inputs, const VariableNameMap& outputs, AttributeMap attrs) { auto& info = OpInfoMap::Instance().Get(type); - if (info.checker_ != nullptr) { - info.checker_->Check(attrs); + if (info.Checker() != nullptr) { + info.Checker()->Check(attrs); } auto op = info.Creator()(type, inputs, outputs, attrs); return std::unique_ptr(op); From 6dcbeb6125fbb0e34ab08a7ebbcc171da01514d8 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Fri, 6 Oct 2017 17:29:43 -0700 Subject: [PATCH 165/170] Update if_else_op.md according to https://github.com/PaddlePaddle/Paddle/issues/4313 --- doc/design/if_else_op.md | 76 +++++++++++++++++++++++----------------- 1 file changed, 43 insertions(+), 33 deletions(-) diff --git a/doc/design/if_else_op.md b/doc/design/if_else_op.md index 954a19c073..26d140f06d 100644 --- a/doc/design/if_else_op.md +++ b/doc/design/if_else_op.md @@ -1,41 +1,51 @@ -IfOp should have only one branch. An IfOp operator takes a `cond` variable whose value must be a vector of N boolean elements. Its return value has N instances. If cond[i] == True, input instance input[i] will go through true_block() and generate output[i]; otherwise it will produce output from false_bloack(). +# The `IfElse` Operator -```python -import paddle as pd +PaddlePaddle's `IfElse` operator differs from TensorFlow's: -x = var() -y = var() -cond = var() -default_value = var() -b = pd.create_ifelseop(inputs=[x], output_num=1) -with b.true_block(): - x = b.inputs(0) - z = operator.add(x, y) - b.set_output(0, operator.softmax(z)) - -with b.false_block(): - x = b.inputs(0) - z = layer.fc(x) - b.set_output(0, operator.softmax(z)) - -out = b(cond) -``` +- the TensorFlow version takes a scalar boolean value as the condition so that the whole mini-batch goes to either the true or the false branch, whereas +- the PaddlePaddle version takes a vector of boolean value as the condition, and instances corresponding to true values go to the true branch, those corresponding to false values go to the false branch. + +## Example + +The following PaddlePaddle program shows the usage of the IfElse operator: -If only true_block is set in an IfElseOp, a special case is that we can have a default value for false as: ```python import paddle as pd -x = var() -y = var() -cond = var() -default_value = var() -b = pd.create_ifelseop(inputs=[x], output_num=1, default_value) - -with b.true_block(): - x = b.inputs(0) - z = operator.add(x, y) - b.set_output(0, operator.softmax(z)) +x = minibatch([10, 20, 30]) # shape=[None, 1] +y = var(1) # shape=[1], value=1 +z = minibatch([10, 20, 30]) # shape=[None, 1] +cond = larger_than(x, 15) # [false, true, true] + +ie = pd.ifelse() +with ie.true_block(): + d = pd.layer.add(x, y) + ie.output(d, pd.layer.softmax(d)) +with ie.false_block(): + d = pd.layer.fc(z) + ie.output(d, d+1) +o1, o2 = ie(cond) +``` -out = b(cond) +A challenge to implement the `IfElse` operator is to infer those variables to be split, or, say, to identify the variable of the mini-batch or those derived from the mini-batch. + +An equivalent C++ program is as follows: + +```c++ +namespace pd = paddle; + +int x = 10; +int y = 1; +int z = 10; +bool cond = false; +int o1, o2; +if (cond) { + int d = x + y; + o1 = z; + o2 = pd::layer::softmax(z); +} else { + int d = pd::layer::fc(z); + o1 = d; + o2 = d+1; +} ``` -where default_value is a list of vars for `cond` == False. From 20a6ae7f1f7eb67eb9f5e7b6290aa81aa4536e0a Mon Sep 17 00:00:00 2001 From: Yan Chunwei Date: Fri, 6 Oct 2017 20:41:37 -0400 Subject: [PATCH 166/170] Feature/tensor array add python binding (#4616) --- paddle/framework/tensor_array.h | 3 + paddle/pybind/CMakeLists.txt | 2 +- paddle/pybind/pybind.cc | 51 +++++++++ .../v2/framework/tests/test_tensor_array.py | 106 ++++++++++++++++++ 4 files changed, 161 insertions(+), 1 deletion(-) create mode 100644 python/paddle/v2/framework/tests/test_tensor_array.py diff --git a/paddle/framework/tensor_array.h b/paddle/framework/tensor_array.h index 22ae6a966f..94a14c2df4 100644 --- a/paddle/framework/tensor_array.h +++ b/paddle/framework/tensor_array.h @@ -26,6 +26,9 @@ namespace framework { * in original lod-tensor. */ struct DySeqMeta { + DySeqMeta(size_t begin, size_t end, size_t ori_idx) + : begin(begin), end(end), ori_idx(ori_idx) {} + size_t begin; size_t end; // not included size_t ori_idx; diff --git a/paddle/pybind/CMakeLists.txt b/paddle/pybind/CMakeLists.txt index 18ecbd1aa3..97364f2db9 100644 --- a/paddle/pybind/CMakeLists.txt +++ b/paddle/pybind/CMakeLists.txt @@ -1,6 +1,6 @@ if(WITH_PYTHON) cc_library(paddle_pybind SHARED SRCS pybind.cc exception.cc protobuf.cc - DEPS pybind python backward proto_desc + DEPS pybind python backward proto_desc tensor_array ${GLOB_OP_LIB}) endif(WITH_PYTHON) diff --git a/paddle/pybind/pybind.cc b/paddle/pybind/pybind.cc index 38ba450447..356c4986e2 100644 --- a/paddle/pybind/pybind.cc +++ b/paddle/pybind/pybind.cc @@ -16,6 +16,7 @@ limitations under the License. */ #include "paddle/framework/backward.h" #include "paddle/framework/lod_tensor.h" +#include "paddle/framework/tensor_array.h" #include "paddle/operators/cond_op.h" #include "paddle/operators/net_op.h" #include "paddle/operators/recurrent_op.h" @@ -286,6 +287,56 @@ All parameter, weight, gradient are variables in Paddle. self->CompleteAddOp(); }); + py::class_(m, "TensorArray") + .def("__init__", + [](TensorArray &instance) { new (&instance) TensorArray(); }) + .def("read", + [](TensorArray &self, size_t index) { return self.Read(index); }) + .def("write", [](TensorArray &self, size_t index, + LoDTensor &value) { self.Write(index, value); }) + .def("write_shared", + [](TensorArray &self, size_t index, const LoDTensor &value) { + self.WriteShared(index, value); + }) + .def("size", [](TensorArray &self) { return self.size(); }) + .def("pack", + [](TensorArray &self, size_t level, + const std::vector> &meta_info, + const std::vector> &lod) { + std::vector meta; + for (auto &info : meta_info) { + PADDLE_ENFORCE_EQ(info.size(), 3UL); + meta.emplace_back(info[0], info[1], info[2]); + } +#ifndef PADDLE_WITH_CUDA + return self.Pack(level, meta, lod); +#else + LoD new_lod; + new_lod.reserve(lod.size()); + std::copy(lod.begin(), lod.end(), std::back_inserter(new_lod)); + return self.Pack(level, meta, new_lod); +#endif + }) + .def("unpack", + [](TensorArray &self, const LoDTensor &source, int level, + bool length_descend) { + auto metas = self.Unpack(source, level, length_descend); + std::vector> meta_info; + for (auto meta : metas) { + meta_info.emplace_back( + std::vector({meta.begin, meta.end, meta.ori_idx})); + } + return meta_info; + }) + .def("stack", [](TensorArray &self) { return self.Stack(); }) + .def("unstack", + [](TensorArray &self, const LoDTensor &source) { + return self.Unstack(source); + }) + .def("unstack_shared", [](TensorArray &self, const LoDTensor &source) { + return self.UnstackShared(source); + }); + // recurrent_op py::class_(m, "RecurrentOp") .def_static( diff --git a/python/paddle/v2/framework/tests/test_tensor_array.py b/python/paddle/v2/framework/tests/test_tensor_array.py new file mode 100644 index 0000000000..11f8a01f92 --- /dev/null +++ b/python/paddle/v2/framework/tests/test_tensor_array.py @@ -0,0 +1,106 @@ +import logging +import paddle.v2.framework.core as core +import unittest +import numpy as np + + +class TestTensorArray(unittest.TestCase): + def setUp(self): + self.ta = core.TensorArray() + + self.batch_size = 10 + self.dim = 2 + + # create a LoDTensor + self.scope = core.Scope() + var = self.scope.new_var("test_tensor") + self.place = core.CPUPlace() + tensor = var.get_tensor() + tensor.set_dims([self.batch_size, self.dim]) + tensor.alloc_float(self.place) + tensor_array = np.array(tensor) + tensor_array[0, 0] = 0 + tensor_array[1, 0] = 1 + tensor_array[2, 0] = 2 + tensor_array[3, 0] = 3 + tensor_array[4, 0] = 4 + tensor_array[5, 0] = 5 + tensor_array[6, 0] = 6 + tensor_array[7, 0] = 7 + tensor_array[8, 0] = 8 + tensor_array[9, 0] = 9 + + lod_py = [[0, 2, 5, 10]] + lod_tensor = core.LoDTensor(lod_py) + lod_tensor.set(tensor_array, self.place) + + self.py_seq_meta = [[5, 10, 2], [2, 5, 1], [0, 2, 0]] + + self.tensor = lod_tensor + + def test_unstack(self): + self.ta.unstack(self.tensor) + self.assertEqual(self.tensor.get_dims()[0], self.ta.size()) + + def test_read(self): + self.ta.unstack(self.tensor) + for i in range(self.batch_size): + tensor = self.ta.read(i) + + def test_write(self): + self.ta.unstack(self.tensor) + + # create a tensor with shape of [1, self.dim] + var = self.scope.new_var("hell") + tensor = var.get_tensor() + tensor.set_dims([1, self.dim]) + tensor.alloc_float(self.place) + tensor_array = np.array(tensor) + for i in range(self.dim): + tensor_array[0, i] = i + tensor.set(tensor_array, self.place) + + self.ta.write(2, tensor) + + ta_tensor = self.ta.read(2) + ta_tensor_array = np.array(ta_tensor) + self.assertEqual(ta_tensor.get_dims(), [1, self.dim]) + self.assertTrue((tensor_array == ta_tensor_array).all()) + + def test_write_shared(self): + self.ta.unstack(self.tensor) + + # create a tensor with shape of [1, self.dim] + var = self.scope.new_var("hell") + tensor = var.get_tensor() + tensor.set_dims([1, self.dim]) + tensor.alloc_float(self.place) + tensor_array = np.array(tensor) + for i in range(self.dim): + tensor_array[0, i] = i + tensor.set(tensor_array, self.place) + + self.ta.write_shared(2, tensor) + + ta_tensor = self.ta.read(2) + ta_tensor_array = np.array(ta_tensor) + self.assertEqual(ta_tensor.get_dims(), [1, self.dim]) + self.assertTrue((tensor_array == ta_tensor_array).all()) + + def test_unpack(self): + meta = self.ta.unpack(self.tensor, 0, True) + self.assertEqual(self.ta.size(), 5) + self.assertEqual(meta, self.py_seq_meta) + + def test_pack(self): + meta = self.ta.unpack(self.tensor, 0, True) + print "meta", meta + tensor = self.ta.pack(0, meta, self.tensor.lod()) + print np.array(self.tensor) + print np.array(tensor) + self.assertTrue((np.array(self.tensor) == np.array(tensor)).all()) + self.assertTrue(tensor.lod(), self.tensor.lod()) + + +if __name__ == '__main__': + unittest.main() From 34193844adc9d67e4a9c8bb00ff9e67ffbc4ba3c Mon Sep 17 00:00:00 2001 From: Yan Chunwei Date: Fri, 6 Oct 2017 21:06:44 -0400 Subject: [PATCH 167/170] add background for TensorArray (#4564) --- doc/design/tensor_array.md | 268 ++++++++++++++++++++++++++++++++----- 1 file changed, 233 insertions(+), 35 deletions(-) diff --git a/doc/design/tensor_array.md b/doc/design/tensor_array.md index a0419ec002..8378e97bf7 100644 --- a/doc/design/tensor_array.md +++ b/doc/design/tensor_array.md @@ -1,39 +1,250 @@ # Design for TensorArray +This design doc presents the necessity of a new C++ class `TensorArray`. +In addition to the very simple C++ implementation + +```c++ +class TensorArray { + public: + explicit TensorArray(const LoDTensor&); + explicit TensorArray(size_t size); + + private: + vector values_; +}; +``` + +We also need to expose it to PaddlePaddle's Python API, +because users would want to use it with our very flexible operators `WhileLoop`. +An example for a RNN based on dynamic operators is + +```python +input = pd.data(...) +num_steps = Var(12) + +TensorArray states(size=num_steps) +TensorArray step_inputs(unstack_from=input) +TensorArray step_outputs(size=num_steps) + +W = Tensor(...) +U = Tensor(...) +default_state = some_op() + +step = Var(1) + +wloop = paddle.create_whileloop(loop_vars=[step]) +with wloop.frame(): + wloop.break_if(pd.equal(step, num_steps) + pre_state = states.read(step-1, default_state) + step_input = step_inputs.read(step) + state = pd.sigmoid(pd.matmul(U, pre_state) + pd.matmul(W, step_input)) + states.write(step, state) + step_outputs.write(step, state) # output state + step.update(state+1) + +output = step_outputs.stack() +``` + +## Background +Steps are one of the core concepts of RNN. In each time step of RNN, there should be several input segments, states, and output segments; all these components act like arrays, for example, call `states[step_id]` will get the state in `step_id`th time step. + +An RNN can be implemented with the following pseudocode + +```c++ +Array states; +Array input_segments; +Array output_segments; +Parameter W, U; + +step = 1 +seq_len = 12 +while_loop { + if (step == seq_len) break; + states[step] = sigmoid(W * states[step-1] + U * input_segments[step]); + output_segments[step] = states[step] // take state as output + step++; +} +``` +According to the [RNN roadmap](https://github.com/PaddlePaddle/Paddle/issues/4561), there are several different RNNs that PaddlePaddle will eventually support. + +Currently, the basic RNN implementation supported by PaddlePaddle is the `recurrent_op` which takes tensors as input and splits them into `input_segments`. + + +Since a tensor cannot store variable-length sequences directly, PaddlePaddle implements the tensor with level of details (`LoDTensor` for short). +Segmenting the `LoDTensor` is much more complicated than splitting a tensor, that makes it necessary to refactor the `recurrent_op` with `LoDTensor` segmenting support. + +As the next step in RNN support, `dynamic_recurrent_op` should be introduced to handle inputs with variable-length sequences. + +The implementation is similar to `recurrent_op`. +The key difference is the way **the original input `LoDTensors` and outupts are split to get the `input_segments` and the `output_segments`.** + + +Though it can't be built over `recurrent_op` or `dynamic_recurrent_op` directly, +the logic behind splitting a tensor or a LoD tensor into `input_segments` remains the same. + +## Why `TensorArray` +The logic behind splitting the inputs to segments, states and outputs is similar and can be shared in a seperate module. + +The array of `states`, `input_segments` and `output_segments` would be exposed to users when writing a dynamic RNN model similar to the above pseudo codes. + +So there should be an array-like container, which can store the segments of a tensor or LoD tensor. + +**This container can store an array of tensors and provides several methods to split a tensor or a LoD tensor** . +This is where the notion of `TensorArray` comes from. + +## Introduce TensorArray to uniform all the three RNNs TensorArray as a new concept is borrowed from TensorFlow, it is meant to be used with dynamic iteration primitives such as `while_loop` and `map_fn`. This concept can be used to support our new design of dynamic operations, and help to refactor some existing variant-sentence-related layers, -such as `RecurrentGradientMachine`. +such as `recurrent_op`, `RecurrentGradientMachine`. In [our design for dynamic RNN](https://github.com/PaddlePaddle/Paddle/pull/4401), `TensorArray` is used to segment inputs and store states in all time steps. By providing some methods similar to a C++ array, -the definition of some state-based dynamic models such as RNN could be more natural and highly flexible. - -## Dynamic-Related Methods -Some basic methods should be proposed as follows: - -### stack() -Pack the values in a `TensorArray` into a tensor with rank one higher than each tensor in `values`. -### unstack(axis=0) -Unpacks the given dimension of a rank-`R` tensor into rank-`(R-1)` tensors. -### concat() -Return the values in the `TensorArray` as a concatenated Tensor. -### write(index, value, data_shared=true) -Write value into index of the TensorArray. -### read(index) -Read the value at location `index` in the `TensorArray`. -### size() -Return the number of values. +the definition of some state-based dynamic models such as RNN can be more natural and highly flexible. + +## Dynamic-operations on TensorArray + +`TensorArray` will be used directly when defining dynamic models, so some operators listed below should be implemented + +```python +# several helper operators for TensorArray +def tensor_array_stack(ta, tensor): + ''' + get a tensor array `ta`, return a packed `tensor`. + ''' + pass + +def tensor_array_unstack(tensor, ta): + ''' + get a `tensor`, unstack it and get a tensor array `ta`. + ''' + pass + +def tensor_array_write(ta, index, tensor, data_shared): + ''' + get a `tensor` and a scalar tensor `index`, write `tensor` into index-th + value of the tensor array `ta`. + `data_shared` is an attribute that specifies whether to copy or reference the tensors. + ''' + pass + +def tensor_array_read(ta, index, tensor): + ''' + get a tensor array `ta`, a scalar tensor `index`, read the index-th value of + `ta` and return as the `tensor`. + ''' + pass + +def tensor_array_size(ta, tensor): + ''' + get a tensor array `ta`, return the size of `ta` and return as the scalar `tensor`. + ''' + pass +``` + +It is trivial for users to use so many low-level operators, so some helper methods should be proposed in python wrapper to make `TensorArray` easier to use, +for example + +```python +class TensorArray: + def __init__(self, name): + self.name = name + self.desc = TensorArrayDesc() + + def stack(self, name=None): + ''' + Pack the values in a `TensorArray` into a tensor with rank one higher + than each tensor in `values`. + `stack` can be used to split tensor into time steps for RNN or whileloop. + + @name: str + the name of the variable to output. + ''' + tensor = NewVar(name) + tensor_array_stack(self.name, tensor) + return tensor + + def unstack(self, input): + ''' + Unpacks the given dimension of a rank-`R` tensor into rank-`(R-1)` tensors. + `unstack` can be used to concatenate all the time steps for RNN or whileloop. + + @input: str + the name of input tensor + ''' + tensor_array_unstack(tensor, self.name) + + def write(self, index, value, data_shared=True): + ''' + Write value into index of the TensorArray. + If `data_shared` is set to True, than the index-th value in TensorArray will + be shared with the tensor passed in. + + @index: str + name of a scalar tensor + @value: str + name of a tensor + @data_shared: bool + ''' + tensor_array_write(self.name, index, value, data_shared) + + def read(self, index, output): + ''' + Read the value at location `index` in the `TensorArray`. + + @index: str + name of a scalar tensor + @output: + name of a output variable + ''' + tensor_array_read(self.name, index, output) + + + def size(self, output): + ''' + Return the number of values. + + @output: str + name of a scalar tensor + ''' + tensor_array_size(self.name, output) +``` ## LoDTensor-related Supports -The `RecurrentGradientMachine` in Paddle serves as a flexible RNN layer; it takes variant length sequences as input, -because each step of RNN could only take a tensor-represented batch of data as input, +The `RecurrentGradientMachine` in Paddle serves as a flexible RNN layer; it takes varience-length sequences as input, and output sequences too. + +Since each step of RNN can only take a tensor-represented batch of data as input, some preprocess should be taken on the inputs such as sorting the sentences by their length in descending order and cut each word and pack to new batches. -Such cut-like operations can be embedded into `TensorArray` as general methods called `unpack` and `pack`. +Such cut-like operations can be embedded into `TensorArray` as general methods called `unpack` and `pack`, +these two operations are similar to `stack` and `unstack` except that they operate on variable-length sequences formated as a LoD tensor rather than a tensor. + +Some definitions are like + +```python +def unpack(level): + ''' + Split LodTensor in some `level` and generate batches, if set `sort_by_length`, + will sort by length. -With these two methods, a variant-sentence-RNN can be implemented like + Returns: + - a new `TensorArray`, whose values are LodTensors and represents batches + of data. + - an int32 Tensor, which stores the map from the new batch's indices to + original LoDTensor + ''' + pass + +def pack(level, indices_map): + ''' + Recover the original LoD-arranged LoDTensor with the values in a `TensorArray` + and `level` and `indices_map`. + ''' + pass +``` + +With these two methods, a varience-length sentence supported RNN can be implemented like ```c++ // input is the varient-length data @@ -58,16 +269,3 @@ LoDTensor rnn_output = ta.pack(ta, indice_map); ``` the code above shows that by embedding the LoDTensor-related preprocess operations into `TensorArray`, the implementation of a RNN that supports varient-length sentences is far more concise than `RecurrentGradientMachine` because the latter mixes all the codes together, hard to read and extend. - - -some details are as follows. - -### unpack(level, sort_by_length) -Split LodTensor in some `level` and generate batches, if set `sort_by_length`, will sort by length. - -Returns: - -- a new `TensorArray`, whose values are LodTensors and represents batches of data. -- an int32 Tensor, which stores the map from the new batch's indices to original LoDTensor -### pack(level, indices_map) -Recover the original LoD-arranged LoDTensor with the values in a `TensorArray` and `level` and `indices_map`. From bfe6dcb5d345b3a734c2d496a97bffa184dfbbd4 Mon Sep 17 00:00:00 2001 From: Yan Chunwei Date: Fri, 6 Oct 2017 23:48:44 -0400 Subject: [PATCH 168/170] fix RNN and IfElse syntax in Block design (#4210) * fix block syntax * add builder * fix argument names * uniform all Var to var * fix c demo * factor set_outputs * update block * delete as_ifelse_input interface from block design * simplify set_outputs and output_num * make the c++ codes compatible with demo * fix compatible * fix syntax * update ie * update syntax --- doc/design/block.md | 96 ++++++++++++++++++++++----------------------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/doc/design/block.md b/doc/design/block.md index be88001220..4d5dd4ba95 100644 --- a/doc/design/block.md +++ b/doc/design/block.md @@ -55,17 +55,23 @@ Let us consolidate the discussion by presenting some examples. The following C++ programs shows how blocks are used with the `if-else` structure: ```c++ +namespace pd = paddle; + int x = 10; -int y = 20; -int out; +int y = 1; +int z = 10; bool cond = false; +int o1, o2; if (cond) { int z = x + y; - out = softmax(z); + o1 = z; + o2 = pd::layer::softmax(z); } else { - int z = fc(x); - out = z; + int d = pd::layer::fc(z); + o1 = d; + o2 = d+1; } + ``` An equivalent PaddlePaddle program from the design doc of the [IfElseOp operator](./if_else_op.md) is as follows: @@ -73,57 +79,55 @@ An equivalent PaddlePaddle program from the design doc of the [IfElseOp operator ```python import paddle as pd -x = var(10) -y = var(20) -cond = var(false) -ie = pd.create_ifelseop(inputs=[x], output_num=1) +x = minibatch([10, 20, 30]) # shape=[None, 1] +y = var(1) # shape=[1], value=1 +z = minibatch([10, 20, 30]) # shape=[None, 1] +cond = larger_than(x, 15) # [false, true, true] + +ie = pd.ifelse() with ie.true_block(): - x = ie.inputs(true, 0) - z = operator.add(x, y) - ie.set_output(true, 0, operator.softmax(z)) + d = pd.layer.add_scalar(x, y) + ie.output(d, pd.layer.softmax(d)) with ie.false_block(): - x = ie.inputs(false, 0) - z = layer.fc(x) - ie.set_output(true, 0, operator.softmax(z)) -out = b(cond) + d = pd.layer.fc(z) + ie.output(d, d+1) +o1, o2 = ie(cond) ``` -In both examples, the left branch computes `softmax(x+y)` and the right branch computes `fc(x)`. +In both examples, the left branch computes `x+y` and `softmax(x+y)`, the right branch computes `x+1` and `fc(x)`. A difference is that variables in the C++ program contain scalar values, whereas those in the PaddlePaddle programs are mini-batches of instances. The `ie.input(true, 0)` invocation returns instances in the 0-th input, `x`, that corresponds to true values in `cond` as the local variable `x`, where `ie.input(false, 0)` returns instances corresponding to false values. + ### Blocks with `for` and `RNNOp` The following RNN model from the [RNN design doc](./rnn.md) ```python -x = sequence([10, 20, 30]) -m = var(0) -W = tensor() -U = tensor() - -rnn = create_rnn(inputs=[input]) -with rnn.stepnet() as net: - x = net.set_inputs(0) - h = net.add_memory(init=m) - fc_out = pd.matmul(W, x) - hidden_out = pd.matmul(U, h.pre(n=1)) - sum = pd.add_two(fc_out, hidden_out) - act = pd.sigmoid(sum) - h.update(act) # update memory with act - net.set_outputs(0, act, hidden_out) # two outputs - +x = sequence([10, 20, 30]) # shape=[None, 1] +m = var(0) # shape=[1] +W = var(0.314, param=true) # shape=[1] +U = var(0.375, param=true) # shape=[1] + +rnn = pd.rnn() +with rnn.step(): + h = rnn.memory(init = m) + hh = rnn.previous_memory(h) + a = layer.fc(W, x) + b = layer.fc(U, hh) + s = pd.add(a, b) + act = pd.sigmoid(s) + rnn.update_memory(h, act) + rnn.output(a, b) o1, o2 = rnn() -print o1, o2 ``` - has its equivalent C++ program as follows ```c++ int* x = {10, 20, 30}; -int m = 0; -int W = some_value(); -int U = some_other_value(); +int* m = {0}; +int* W = {0.314}; +int* U = {0.375}; int mem[sizeof(x) / sizeof(x[0]) + 1]; int o1[sizeof(x) / sizeof(x[0]) + 1]; @@ -131,20 +135,16 @@ int o2[sizeof(x) / sizeof(x[0]) + 1]; for (int i = 1; i <= sizeof(x)/sizeof(x[0]); ++i) { int x = x[i-1]; if (i == 1) mem[0] = m; - int fc_out = W * x; - int hidden_out = Y * mem[i-1]; - int sum = fc_out + hidden_out; + int a = W * x; + int b = Y * mem[i-1]; + int s = fc_out + hidden_out; int act = sigmoid(sum); mem[i] = act; o1[i] = act; o2[i] = hidden_out; } - -print_array(o1); -print_array(o2); ``` - ## Compilation and Execution Like TensorFlow programs, a PaddlePaddle program is written in Python. The first part describes a neural network as a protobuf message, and the rest part executes the message for training or inference. @@ -210,11 +210,11 @@ a = pd.Varaible(shape=[20, 20]) b = pd.fc(a, params=["fc.w", "fc.b"]) rnn = pd.create_rnn() -with rnn.stepnet() as net: - x = net.set_inputs(a) +with rnn.stepnet() + x = a.as_step_input() # reuse fc's parameter fc_without_b = pd.get_variable("fc.w") - net.set_outputs(fc_without_b) + rnn.output(fc_without_b) out = rnn() ``` From cb1baa3be739023a0a72dc36a45c71f4fc4545f4 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Fri, 6 Oct 2017 22:54:35 -0700 Subject: [PATCH 169/170] Update program.md (#4618) * Update program.md * Update * Update --- doc/design/program.md | 104 +++++++++++++++++++++++++++---- paddle/framework/framework.proto | 3 + 2 files changed, 94 insertions(+), 13 deletions(-) diff --git a/doc/design/program.md b/doc/design/program.md index fb8f86ac07..bd2456787c 100644 --- a/doc/design/program.md +++ b/doc/design/program.md @@ -1,8 +1,10 @@ -# Design Doc: ProgramDesc +# Design Doc: PaddlePaddle Programs -The basic structure of a PaddlePaddle program is some nested blocks, as a C++ or Java program. +## Compile and Execution + +A PaddlePaddle program consists of two parts -- the first generates a `ProgramDesc` protobuf message that describes the program, and the second runs this message using a C++ class `Executor`. -As described in [graph.md](./graph.md), the first five lines of the following PaddlePaddle program +A simple example PaddlePaddle program can be found in [graph.md](./graph.md): ```python x = layer.data("images") @@ -13,36 +15,112 @@ optimize(cost) train(cost, reader=mnist.train()) ``` -generates, or compiles, a PaddelPaddle program, which is represented by the following protobuf message: +The first five lines of the following PaddlePaddle program generates, or, compiles, the `ProgramDesc` message. The last line runs it. -```protobuf -message ProgramDesc { - repeated BlockDesc blocks = 1; +## Programs and Blocks + +The basic structure of a PaddlePaddle program is some nested blocks, as a C++ or Java program. + +- program: some nested blocks +- [block](./block.md): + - some local variable definitions, and + - a sequence of operators + +The concept of block comes from usual programs. For example, the following C++ program has three blocks: + +```c++ +int main() { // block 0 + int i = 0; + if (i < 10) { // block 1 + for (int j = 0; j < 10; j++) { // block 2 + } + } + return 0; } +``` + +The following PaddlePaddle program has three blocks: + +```python +import paddle as pd // block 0 + +x = minibatch([10, 20, 30]) # shape=[None, 1] +y = var(1) # shape=[1], value=1 +z = minibatch([10, 20, 30]) # shape=[None, 1] +cond = larger_than(x, 15) # [false, true, true] +ie = pd.ifelse() +with ie.true_block(): // block 1 + d = pd.layer.add_scalar(x, y) + ie.output(d, pd.layer.softmax(d)) +with ie.false_block(): // block 2 + d = pd.layer.fc(z) + ie.output(d, d+1) +o1, o2 = ie(cond) +``` + +## `BlockDesc` and `ProgramDesc` + +All protobuf messages are defined in `framework.proto`. + +`BlockDesc` is straight-forward -- it includes local variable definitions, `vars`, and a sequence of operators, `ops`. + +```protobuf message BlockDesc { required int32 parent = 1; repeated VarDesc vars = 2; repeated OpDesc ops = 3; } +``` + +The parent ID indicates the parent block so that operators in a block can refer to variables defined locally and also those defined in their ancestor blocks. + +All hierarchical blocks in a program are flattened and stored in an array. The block ID is the index of the block in this array. + +```protobuf +message ProgramDesc { + repeated BlockDesc blocks = 1; +} +``` + + +### Global Block +The global block is the first one in the above array. + +## Operators that Use Blocks + +In the above example, the operator `IfElseOp` has two blocks -- the true branch and the false branch. + +The definition of `OpDesc` shows that an operator could have some attributes: + +```protobuf message OpDesc { AttrDesc attrs = 1; ... } +``` + +and an attribute could be of type block, which is, in fact, a block ID as described above: +``` message AttrDesc { - required AttrType type = 1; + required string name = 1; - // index into ProgramDesc::blocks when type==BLOCK - optional int32 block = 2; + enum AttrType { + INT = 1, + STRING = 2, + ... + BLOCK = ... + } + required AttrType type = 2; + + optional int32 block = 10; // when type == BLOCK ... } ``` -When each of the first five lines runs, related Python function, e.g., `layer.fc`, calls C++ InferShape functions. This InferShape function needs to access the properties of VarDesc's accessed by the current OpDesc. These VarDesc's might not be defined in the current block, but in some ancestor blocks. This requires that we can trace the parent of a block. - -A nested block is often an attribute of an operator, most likely, an IfElseOp or a WhileOp. In above solution, all blocks are in `ProgramDesc::blocks`, this implicitly assigns a zero-based ID to each block -- the index of the block in `ProgramDesc::blocks`. So that `AttrDesc::block` could be an integer block ID. +## InferShape With this design, the InferShape function should take the following parameters: diff --git a/paddle/framework/framework.proto b/paddle/framework/framework.proto index 9a3077fe6e..ac2827e547 100644 --- a/paddle/framework/framework.proto +++ b/paddle/framework/framework.proto @@ -115,4 +115,7 @@ message BlockDesc { repeated OpDesc ops = 4; } +// Please refer to +// https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/program.md +// for more details. message ProgramDesc { repeated BlockDesc blocks = 1; } From 3f874143fe62062607f341f2559840fc23f4bbd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=AD=A6=E6=AF=85?= Date: Mon, 9 Oct 2017 14:55:03 +0800 Subject: [PATCH 170/170] fix grad debug event (#4536) --- python/paddle/v2/event.py | 14 +++++++++++++- python/paddle/v2/trainer.py | 9 +++++++-- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/python/paddle/v2/event.py b/python/paddle/v2/event.py index e66bf67d79..a0ffd31c54 100644 --- a/python/paddle/v2/event.py +++ b/python/paddle/v2/event.py @@ -10,7 +10,8 @@ There are: * EndPass """ __all__ = [ - 'EndIteration', 'BeginIteration', 'BeginPass', 'EndPass', 'TestResult' + 'EndIteration', 'BeginIteration', 'BeginPass', 'EndPass', 'TestResult', + 'EndForwardBackward' ] @@ -73,6 +74,17 @@ class BeginIteration(object): self.batch_id = batch_id +class EndForwardBackward(object): + """ + Event On One Batch ForwardBackward Complete. + """ + + def __init__(self, pass_id, batch_id, gm): + self.pass_id = pass_id + self.batch_id = batch_id + self.gm = gm + + class EndIteration(WithMetric): """ Event On One Batch Training Complete. diff --git a/python/paddle/v2/trainer.py b/python/paddle/v2/trainer.py index ca95ef13bd..076e755939 100644 --- a/python/paddle/v2/trainer.py +++ b/python/paddle/v2/trainer.py @@ -164,11 +164,18 @@ class SGD(object): pass_type) self.__gradient_machine__.eval(pass_evaluator) self.__gradient_machine__.eval(batch_evaluator) + event_handler( + v2_event.EndForwardBackward( + pass_id=pass_id, + batch_id=batch_id, + gm=self.__gradient_machine__)) for each_param in self.__gradient_machine__.getNonStaticParameters( ): self.__parameter_updater__.update(each_param) cost_sum = out_args.sum() cost = cost_sum / len(data_batch) + self.__parameter_updater__.finishBatch(cost) + batch_evaluator.finish() event_handler( v2_event.EndIteration( pass_id=pass_id, @@ -176,8 +183,6 @@ class SGD(object): cost=cost, evaluator=batch_evaluator, gm=self.__gradient_machine__)) - self.__parameter_updater__.finishBatch(cost) - batch_evaluator.finish() self.__parameter_updater__.finishPass() pass_evaluator.finish()