diff --git a/CMakeLists.txt b/CMakeLists.txt
index 4783095194..1252e75398 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -105,6 +105,12 @@ if (WITH_C_API AND WITH_PYTHON)
"different Python interpreter from compiling.")
endif()
+if(MOBILE_INFERENCE)
+ set(THIRD_PARTY_BUILD_TYPE MinSizeRel)
+else()
+ set(THIRD_PARTY_BUILD_TYPE Release)
+endif()
+
########################################################################################
include(external/mklml) # download mklml package
diff --git a/cmake/external/eigen.cmake b/cmake/external/eigen.cmake
index f7483f6be9..bd853d921b 100644
--- a/cmake/external/eigen.cmake
+++ b/cmake/external/eigen.cmake
@@ -8,7 +8,7 @@ ExternalProject_Add(
extern_eigen3
${EXTERNAL_PROJECT_LOG_ARGS}
GIT_REPOSITORY "https://github.com/RLovelett/eigen.git"
- GIT_TAG "master"
+ GIT_TAG 4e79cb69b9425f5f8c3a84be4350d4ab75b5fd9d
PREFIX ${EIGEN_SOURCE_DIR}
UPDATE_COMMAND ""
CONFIGURE_COMMAND ""
diff --git a/cmake/external/gflags.cmake b/cmake/external/gflags.cmake
index 957f8271e4..c819eb4d70 100644
--- a/cmake/external/gflags.cmake
+++ b/cmake/external/gflags.cmake
@@ -36,6 +36,7 @@ ExternalProject_Add(
# change this back to the official Github repo once my PR is
# merged.
GIT_REPOSITORY "https://github.com/wangkuiyi/gflags.git"
+ GIT_TAG 986964c07427ecb9cdb5bd73f73ebbd40e54dadb
PREFIX ${GFLAGS_SOURCES_DIR}
UPDATE_COMMAND ""
CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
@@ -45,11 +46,11 @@ ExternalProject_Add(
-DCMAKE_INSTALL_PREFIX=${GFLAGS_INSTALL_DIR}
-DCMAKE_POSITION_INDEPENDENT_CODE=ON
-DBUILD_TESTING=OFF
- -DCMAKE_BUILD_TYPE=Release
+ -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
${EXTERNAL_OPTIONAL_ARGS}
CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${GFLAGS_INSTALL_DIR}
-DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
- -DCMAKE_BUILD_TYPE:STRING=Release
+ -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
)
ADD_LIBRARY(gflags STATIC IMPORTED GLOBAL)
diff --git a/cmake/external/glog.cmake b/cmake/external/glog.cmake
index b3fef738cc..08bdc1e162 100644
--- a/cmake/external/glog.cmake
+++ b/cmake/external/glog.cmake
@@ -31,6 +31,7 @@ ExternalProject_Add(
${EXTERNAL_PROJECT_LOG_ARGS}
DEPENDS gflags
GIT_REPOSITORY "https://github.com/google/glog.git"
+ GIT_TAG v0.3.5
PREFIX ${GLOG_SOURCES_DIR}
UPDATE_COMMAND ""
CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
@@ -43,12 +44,12 @@ ExternalProject_Add(
-DWITH_GFLAGS=ON
-Dgflags_DIR=${GFLAGS_INSTALL_DIR}/lib/cmake/gflags
-DBUILD_TESTING=OFF
- -DCMAKE_BUILD_TYPE=Release
+ -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
${EXTERNAL_OPTIONAL_ARGS}
CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${GLOG_INSTALL_DIR}
-DCMAKE_INSTALL_LIBDIR:PATH=${GLOG_INSTALL_DIR}/lib
-DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
- -DCMAKE_BUILD_TYPE:STRING=Release
+ -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
)
ADD_LIBRARY(glog STATIC IMPORTED GLOBAL)
diff --git a/cmake/external/gtest.cmake b/cmake/external/gtest.cmake
index 6a2a79b763..5a4aa7a5b7 100644
--- a/cmake/external/gtest.cmake
+++ b/cmake/external/gtest.cmake
@@ -56,11 +56,11 @@ IF(WITH_TESTING)
-DBUILD_GMOCK=ON
-Dgtest_disable_pthreads=ON
-Dgtest_force_shared_crt=ON
- -DCMAKE_BUILD_TYPE=Release
+ -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
${EXTERNAL_OPTIONAL_ARGS}
CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${GTEST_INSTALL_DIR}
-DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
- -DCMAKE_BUILD_TYPE:STRING=Release
+ -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
)
ADD_LIBRARY(gtest STATIC IMPORTED GLOBAL)
diff --git a/cmake/external/protobuf.cmake b/cmake/external/protobuf.cmake
index 7cf7ba85cc..be7f6a9465 100644
--- a/cmake/external/protobuf.cmake
+++ b/cmake/external/protobuf.cmake
@@ -191,12 +191,12 @@ FUNCTION(build_protobuf TARGET_NAME BUILD_FOR_HOST)
${OPTIONAL_ARGS}
-Dprotobuf_BUILD_TESTS=OFF
-DCMAKE_POSITION_INDEPENDENT_CODE=ON
- -DCMAKE_BUILD_TYPE=Release
+ -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
-DCMAKE_INSTALL_PREFIX=${PROTOBUF_INSTALL_DIR}
-DCMAKE_INSTALL_LIBDIR=lib
CMAKE_CACHE_ARGS
-DCMAKE_INSTALL_PREFIX:PATH=${PROTOBUF_INSTALL_DIR}
- -DCMAKE_BUILD_TYPE:STRING=Release
+ -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
-DCMAKE_VERBOSE_MAKEFILE:BOOL=OFF
-DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
${OPTIONAL_CACHE_ARGS}
diff --git a/cmake/external/warpctc.cmake b/cmake/external/warpctc.cmake
index bb258c7b55..8bd0582228 100644
--- a/cmake/external/warpctc.cmake
+++ b/cmake/external/warpctc.cmake
@@ -35,6 +35,7 @@ ExternalProject_Add(
extern_warpctc
${EXTERNAL_PROJECT_LOG_ARGS}
GIT_REPOSITORY "https://github.com/gangliao/warp-ctc.git"
+ GIT_TAG b63a0644654a3e0ed624c85a1767bc8193aead09
PREFIX ${WARPCTC_SOURCES_DIR}
UPDATE_COMMAND ""
CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
@@ -48,9 +49,9 @@ ExternalProject_Add(
-DCMAKE_DISABLE_FIND_PACKAGE_Torch=ON
-DBUILD_SHARED=ON
-DCMAKE_POSITION_INDEPENDENT_CODE=ON
- -DCMAKE_BUILD_TYPE=Release
+ -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
${EXTERNAL_OPTIONAL_ARGS}
- CMAKE_CACHE_ARGS -DCMAKE_BUILD_TYPE:STRING=Release
+ CMAKE_CACHE_ARGS -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
-DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
-DCMAKE_INSTALL_PREFIX:PATH=${WARPCTC_INSTALL_DIR}
)
diff --git a/cmake/external/zlib.cmake b/cmake/external/zlib.cmake
index c496a52b78..e2c9fe56f3 100644
--- a/cmake/external/zlib.cmake
+++ b/cmake/external/zlib.cmake
@@ -42,11 +42,11 @@ ExternalProject_Add(
-DBUILD_SHARED_LIBS=OFF
-DCMAKE_POSITION_INDEPENDENT_CODE=ON
-DCMAKE_MACOSX_RPATH=ON
- -DCMAKE_BUILD_TYPE=Release
+ -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
${EXTERNAL_OPTIONAL_ARGS}
CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${ZLIB_INSTALL_DIR}
-DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
- -DCMAKE_BUILD_TYPE:STRING=Release
+ -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
)
LIST(APPEND external_project_dependencies zlib)
diff --git a/cmake/generic.cmake b/cmake/generic.cmake
index ff9868fc4e..c311783aa3 100644
--- a/cmake/generic.cmake
+++ b/cmake/generic.cmake
@@ -389,13 +389,60 @@ function(go_test TARGET_NAME)
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
endfunction(go_test)
+# Modification of standard 'protobuf_generate_cpp()' with protobuf-lite support
+# Usage:
+# paddle_protobuf_generate_cpp(<proto_srcs> <proto_hdrs> <proto_files>)
+
+function(paddle_protobuf_generate_cpp SRCS HDRS)
+ if(NOT ARGN)
+ message(SEND_ERROR "Error: paddle_protobuf_generate_cpp() called without any proto files")
+ return()
+ endif()
+
+ set(${SRCS})
+ set(${HDRS})
+
+ if (MOBILE_INFERENCE)
+ set(EXTRA_FLAG "lite:")
+ else()
+ set(EXTRA_FLAG "")
+ endif()
+
+ foreach(FIL ${ARGN})
+ get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
+ get_filename_component(FIL_WE ${FIL} NAME_WE)
+
+ set(_protobuf_protoc_src "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc")
+ set(_protobuf_protoc_hdr "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h")
+ list(APPEND ${SRCS} "${_protobuf_protoc_src}")
+ list(APPEND ${HDRS} "${_protobuf_protoc_hdr}")
+
+ add_custom_command(
+ OUTPUT "${_protobuf_protoc_src}"
+ "${_protobuf_protoc_hdr}"
+
+ COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_CURRENT_BINARY_DIR}"
+ COMMAND ${PROTOBUF_PROTOC_EXECUTABLE}
+ -I${CMAKE_CURRENT_SOURCE_DIR}
+ --cpp_out "${EXTRA_FLAG}${CMAKE_CURRENT_BINARY_DIR}" ${ABS_FIL}
+ DEPENDS ${ABS_FIL} protoc
+ COMMENT "Running C++ protocol buffer compiler on ${FIL}"
+ VERBATIM )
+ endforeach()
+
+ set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE)
+ set(${SRCS} ${${SRCS}} PARENT_SCOPE)
+ set(${HDRS} ${${HDRS}} PARENT_SCOPE)
+endfunction()
+
+
function(proto_library TARGET_NAME)
set(oneValueArgs "")
set(multiValueArgs SRCS DEPS)
cmake_parse_arguments(proto_library "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
set(proto_srcs)
set(proto_hdrs)
- protobuf_generate_cpp(proto_srcs proto_hdrs ${proto_library_SRCS})
+ paddle_protobuf_generate_cpp(proto_srcs proto_hdrs ${proto_library_SRCS})
cc_library(${TARGET_NAME} SRCS ${proto_srcs} DEPS ${proto_library_DEPS} protobuf)
endfunction()
diff --git a/doc/design/images/graph_construction_example.dot b/doc/design/images/graph_construction_example.dot
index 8d1b673abf..e115f9844b 100644
--- a/doc/design/images/graph_construction_example.dot
+++ b/doc/design/images/graph_construction_example.dot
@@ -33,7 +33,6 @@ digraph ImageClassificationGraph {
cost -> MSE_Grad [color=red];
d_cost -> MSE_Grad [color=red];
- x -> MSE_Grad [color=red];
l -> MSE_Grad [color=red];
y -> MSE_Grad -> d_y [color=red];
diff --git a/doc/design/images/graph_construction_example_all.png b/doc/design/images/graph_construction_example_all.png
index 1811875034..261611a572 100644
Binary files a/doc/design/images/graph_construction_example_all.png and b/doc/design/images/graph_construction_example_all.png differ
diff --git a/doc/design/images/graph_construction_example_forward_backward.png b/doc/design/images/graph_construction_example_forward_backward.png
index 3049a9315f..4c69687f4a 100644
Binary files a/doc/design/images/graph_construction_example_forward_backward.png and b/doc/design/images/graph_construction_example_forward_backward.png differ
diff --git a/doc/design/images/graph_construction_example_forward_only.png b/doc/design/images/graph_construction_example_forward_only.png
index 25d19088cb..e668c16e0c 100644
Binary files a/doc/design/images/graph_construction_example_forward_only.png and b/doc/design/images/graph_construction_example_forward_only.png differ
diff --git a/doc/design/optimizer.md b/doc/design/optimizer.md
new file mode 100644
index 0000000000..17440fae50
--- /dev/null
+++ b/doc/design/optimizer.md
@@ -0,0 +1,105 @@
+## Optimizer Design
+
+### The Problem
+
+A PaddlePaddle program, or a block, is a sequence of operators operating variables. A training program needs to do three kinds of works:
+
+1. the forward pass, which computes intermediate results and the cost(s),
+1. the backward pass, which derives gradients from intermediate results and costs, and
+1. the optimization pass, which update model parameters to optimize the cost(s).
+
+These works rely on three kinds of operators:
+
+1. forward operators,
+1. gradient operators, and
+1. optimization operators.
+
+It's true that users should be able to create all these operators manually by calling some low-level API, but it would be much more convenient if they could only describe the forward pass and let PaddlePaddle create the backward and optimization operators automatically.
+
+In this design, we propose a high-level API that automatically derives the optimisation pass and operators from the forward pass.
+
+
+### High-level Python API to describe the training process
+
+1. User write code to describe the network:
+
+ ```python
+ images = layer.data("images")
+ labels = layer.data("labels")
+ w1 = pd.var("w1")
+ b1 = pd.var("b1")
+ hidden = layer.fc(images, w=w1, b=b1)
+ cost = layer.mse(hidden, labels)
+ ```
+
+ The above code snippet will create forward operators in [Block](https://github.com/PaddlePaddle/Paddle/blob/develop/doc/design/block.md).
+
+
+2. Users create a certain kind of Optimizer with some argument.
+
+ ```python
+ optimizer = AdagradOptimizer(learing_rate=0.001)
+ ```
+
+3. Users use the optimizer to `minimize` a certain `cost` through updating parameters in parameter_list.
+
+ ```python
+ opt_op_list = optimizer.minimize(cost, parameter_list=[w1, b1])
+ ```
+ The above code snippet will create gradient and optimization operators in Block. The return value of `minimize()` is list of optimization operators that will be run by session.
+
+4. Users use Session/Executor to run this opt_op_list as target to do training.
+
+ ```python
+ sess.run(target= opt_op_list, ...)
+ ```
+
+#### Optimizer Python interface:
+
+```python
+class Optimizer(object):
+ """Optimizer Base class.
+
+ """
+
+ def __init__(self):
+ pass
+
+ def create_backward_pass(self, loss, parameter_list=None):
+ """
+ create and add gradient Operators in BlockDesc to Compute gradients of `loss`
+ for parameters in parameter_list
+
+ Args:
+ loss: an variable generated by cost function.
+ parameter_list: parameters that need to compute gradient and update to optimize the lost.
+
+ Returns:
+ list of (parameters, gradients) pair.
+ """
+ return None
+
+ def create_optimization_pass(self, parameters_and_grads):
+ """Add optimization operators to update gradients to variables.
+
+ Args:
+ parameters_and_grads: a list of (variable, gradient) pair to update.
+
+ Returns:
+ optmization_op_list: a list of optimization operator that will update parameter using gradient.
+ """
+ return None
+
+ def minimize(self, loss, parameter_list):
+ """Add operations to minimize `loss` by updating `parameter_list`.
+
+ This method combines interface `create_backward_pass()` and
+ `create_optimization_pass()` into one.
+ """
+ params_grads = self.create_backward_pass(loss, parameter_list)
+ update_ops = self.create_optimization_pass(params_grads)
+ return update_ops
+
+```
+
+Users can inherit the Optimizer above to create their own Optimizer with some special logic, such as AdagradOptimizer.
diff --git a/doc/design/python_api.md b/doc/design/python_api.md
index c4665e44fc..56ae1d925a 100644
--- a/doc/design/python_api.md
+++ b/doc/design/python_api.md
@@ -214,3 +214,7 @@ def fc_layer(input, size, ...):
out.writer = op
return out
```
+
+## Optimizer
+
+[Optimizer Design Doc](./optimizer.md)
diff --git a/doc/design/selected_rows.md b/doc/design/selected_rows.md
new file mode 100644
index 0000000000..9e6f3b20cb
--- /dev/null
+++ b/doc/design/selected_rows.md
@@ -0,0 +1,74 @@
+# Design Doc: Selected Rows
+
+`SelectedRows` is a kind of sparse tensor data type, which is designed to support `embedding` operators. The gradient of embedding table is a sparse tensor. Only a few rows are non-zero values in that tensor. It is straightforward to represent the sparse tensor by the following sparse tensor data structure:
+
+```cpp
+class SelectedRows {
+ private:
+ vector<int> rows_;
+ Tensor value_;
+ int height_;
+};
+```
+
+The field `height_` shows the first dimension of `SelectedRows`. The `rows` are the indices of which rows of `SelectedRows` are non-zeros. The `value_` field is an N-dim tensor and shape is `[rows.size() /* NUM_ROWS */, ...]`, which supplies values for each row. The dimension of `SelectedRows` satisfies `[height_] + value_.shape[1:]`.
+
+Suppose that a SelectedRows-typed variable `x` has many rows, but only two of them have values -- row 73 is `[1, 2]` and row 84 is `[3, 4]`, the `SelectedRows` representation would be:
+
+```
+x = SelectedRow {
+ rows = [73, 84],
+ value = [[1, 2], [3,4]]
+}
+```
+
+
+## SelectedRows in Protobuf
+
+`SelectedRows` is a kind of `Variable`. `VarDesc` in protobuf should describe the `SelectedRows` information. Only the tensor dimension of a `SelectedRows` will be described in compile-time since the `rows_` and `value_` are related to training data.
+So we use `TensorDesc` to unify `data_type` and `dims`. A LodTensorDesc contains a `TensorDesc` and `lod_level`. The description of `SelectedRows` is a Tensor description.
+
+```proto
+message TensorDesc {
+ required DataType data_type = 1;
+ repeated int64 dims = 2; // [UNK, 640, 480] is saved as [-1, 640, 480]
+}
+
+message LodTensorDesc {
+ required TensorDesc tensor = 1;
+ optional int lod_level = 2;
+}
+
+message VarDesc {
+ required string name = 1;
+ enum VarType {
+ LOD_TENSOR = 0;
+ SELECTED_ROWS = 1;
+ }
+ required VarType type = 2;
+ optional LodTensorDesc lod_desc = 3;
+ optional TensorDesc selected_rows_desc = 4;
+ optional bool persistable = 5 [ default = false ];
+}
+```
+
+## InferShape for Selected Rows
+
+Just like `LoD` information, `InferShape` method will inference output tensor type as well. The operator should decide whether its output is a `SelectedRows` or `Dense` tensor.
+
+For example, the gradient operator of `TableLookup` will always generate `SelectedRows`. Its `InferShape` method should be like following
+
+```cpp
+void TableLookupGrad::InferShape(context) {
+ ...
+ context.SetDataType("Embedding.Grad", kSelectedRows);
+}
+```
+
+
+## Sparse Operators
+
+There are several operators should be written to support `SelectedRows`. They are:
+
+1. Operators which generates `SelectedRows` gradient. e.g. Gradient of `TableLookupOp`.
+2. Optimize operators which support `SelectedRows` gradient. e.g. `SGD` or `AdaGrad` for `SelectedRows`. However, there should be only one `SGD` operator. `OpWithKernel::Run` should select a suitable kernel for both `dense` tensor or `SelectedRows`.
diff --git a/doc/design/var_desc.md b/doc/design/var_desc.md
index bfbbdd0578..0b2958c1b1 100644
--- a/doc/design/var_desc.md
+++ b/doc/design/var_desc.md
@@ -16,16 +16,23 @@ The computation graph is constructed by Data Node and Operation Node. The concep
## Definition of VarDesc
-A VarDesc should have a name and value, in PaddlePaddle, the value will always be a tensor. Since we use LoDTensor most of the time. We add a LoDTesnorDesc to represent it.
+A VarDesc should have a name, and value. The are two kinds of variable type in compile time, they are `LoDTensor` and `SelectedRows`.
```proto
message VarDesc {
required string name = 1;
- optional LoDTensorDesc lod_tensor = 2;
+ enum VarType {
+ LOD_TENSOR = 0;
+ SELECTED_ROWS = 1;
+ }
+ required VarType type = 2;
+ optional LoDTensorDesc lod_desc = 3;
+ optional TensorDesc selected_rows_desc = 4;
+ optional bool persistable = 5 [ default = false ];
}
```
-## Definition of LodTensorDesc
+## Definition of TensorDesc
```proto
enum DataType {
@@ -38,87 +45,25 @@ enum DataType {
FP64 = 6;
}
-message LoDTensorDesc {
+message TensorDesc {
required DataType data_type = 1;
- repeated int32 dims = 2; // [UNK, 640, 480] is saved as [-1, 640, 480]
- optional int32 lod_level = 3 [default=0];
+ repeated int64 dims = 2; // [UNK, 640, 480] is saved as [-1, 640, 480]
}
```
-## Definition of Variable in Python
-
-In Python API, layer will take Variable as Input, and return Variable as Output. There should be a class `Variable` in python to help create and manage Variable.
-
-```python
-image = Variable(dims=[-1, 640, 480])
-# fc1 and fc2 are both Variable
-fc1 = layer.fc(input=image, output_size=10)
-fc2 = layer.fc(input=fc1, output_size=20)
-```
-### what should class `Variable` Have
-1. `name`.a name of string type is used to mark the value of the Variable.
-1. `initializer`. Since our Tensor does not have value. we will always use some Operator to fullfill it when run. So we should have a initialize method to help add the init operator.
-1. `operator`. Variable should record which operator produce itself. The reaon is:
- - we use pd.eval(targets=[var1, var2]) to run the related ops to get the value of var1 and var2. var.op is used to trace the dependency of the current variable.
-
-In PaddlePaddle, we use Block to describe Computation Graph, so in the code we will use Block but not Graph.
-
-```python
-import VarDesc
-import LoDTensorDesc
-import framework
-
-def AddInitialOperator(variable, initializer):
- # add an initialize Operator to block to init this Variable
-
-class Variable(object):
- def __init__(self, name, dims, type, initializer):
- self._block = get_default_block()
- self._name = name
- self.op = None
-
- tensor_desc = LoDTensorDesc(data_type=type, dims=dims)
- _var_desc = VarDesc(name=name, lod_tensor=tensor_desc)
- self._var = framework.CreateVar(_var_desc)
- self._block.add_var(self)
+A TensorDesc describes `SelectedRows` and `LoDTensor`. For details of `SelectedRows`, please reference [`SelectedRows`](./selected_rows.md).
- # add initial op according to initializer
- if initializer is not None:
- AddInitialOperator(self, initializer)
-
- def dims(self):
- return self._var.dims()
-
- def data_type(self):
- return self._var.data_type()
+## Definition of LodTensorDesc
- def to_proto(self):
- pass
+```proto
+message LoDTensorDesc {
+ required TensorDesc tensor = 1;
+ optional int lod_level = 2;
+}
```
-Then we can use this Variable to create a fc layer in Python.
+A LoDTensorDesc contains a tensor and a lod_level.
-```python
-import paddle as pd
-
-def flatten_size(X, num_flatten_dims):
- prod = 1 # of last num_flatten_dims
- for i in xrange(num_flatten_dims):
- prod = prod * X.dims[-i-1]
- return prod
-
-def layer.fc(X, output_size, num_flatten_dims):
- W = Variable(pd.random_uniform(), type=FP32, dims=[flatten_size(X, num_flatten_dims), output_size])
- b = Variable(pd.random_uniform(), type=FP32, dims=[output_size])
- out = Variable(type=FP32)
- y = operator.fc(X, W, b, output=out) # fc will put fc op input into out
- pd.InferShape(y)
- return out
-
-x = Variable(dims=[-1, 640, 480])
-y = layer.fc(x, output_size=100)
-z = layer.fc(y, output_size=200)
+## Definition of Variable in Python
-paddle.eval(targets=[z], ...)
-print(z)
-```
+For Variable in Python, please reference [`Python API`](./python_api.md).
diff --git a/paddle/api/CMakeLists.txt b/paddle/api/CMakeLists.txt
index d7b3d2bdec..d6b8464100 100644
--- a/paddle/api/CMakeLists.txt
+++ b/paddle/api/CMakeLists.txt
@@ -26,7 +26,7 @@ FILE(GLOB PY_PADDLE_PYTHON_FILES ${PADDLE_SOURCE_DIR}/paddle/py_paddle/*.py)
SET_SOURCE_FILES_PROPERTIES(Paddle.i PROPERTIES CPLUSPLUS ON)
SET(CMAKE_SWIG_OUTDIR ${CMAKE_CURRENT_BINARY_DIR})
-SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-parentheses-equality -Wno-missing-field-initializers -Wno-self-assign")
+SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-parentheses-equality -Wno-missing-field-initializers -Wno-self-assign -ftls-model=global-dynamic")
SET(SWIG_MODULE_swig_paddle_EXTRA_DEPS
paddle_parameter
diff --git a/paddle/framework/CMakeLists.txt b/paddle/framework/CMakeLists.txt
index 1bf80b3e58..14947b6f21 100644
--- a/paddle/framework/CMakeLists.txt
+++ b/paddle/framework/CMakeLists.txt
@@ -19,10 +19,10 @@ cc_test(scope_test SRCS scope_test.cc DEPS scope)
proto_library(framework_proto SRCS framework.proto)
cc_library(attribute SRCS attribute.cc DEPS framework_proto)
-cc_library(proto_desc SRCS var_desc.cc op_desc.cc block_desc.cc program_desc.cc DEPS attribute ddim)
+cc_library(proto_desc SRCS var_desc.cc op_desc.cc block_desc.cc program_desc.cc DEPS attribute ddim op_info)
cc_library(op_proto_maker SRCS op_proto_maker.cc DEPS framework_proto attribute)
cc_test(op_proto_maker_test SRCS op_proto_maker_test.cc DEPS op_proto_maker)
-cc_library(op_info SRCS op_info.cc DEPS attribute framework_proto proto_desc)
+cc_library(op_info SRCS op_info.cc DEPS attribute framework_proto)
cc_library(operator SRCS operator.cc DEPS op_info device_context tensor scope proto_desc)
cc_test(operator_test SRCS operator_test.cc DEPS operator op_registry)
@@ -42,5 +42,14 @@ add_custom_command(TARGET framework_py_proto POST_BUILD
cc_library(backward SRCS backward.cc DEPS net_op)
cc_test(backward_test SRCS backward_test.cc DEPS backward recurrent_op device_context)
+cc_library(executor SRCS executor.cc DEPS op_registry device_context scope framework_proto backward)
+set(EXECUTOR_TEST_OP elementwise_add_op gaussian_random_op feed_op fetch_op
+ mul_op sum_op squared_l2_distance_op fill_constant_op sgd_op)
+if(WITH_GPU)
+ nv_test(executor_test SRCS executor_test.cc DEPS executor ${EXECUTOR_TEST_OP})
+else()
+ cc_test(executor_test SRCS executor_test.cc DEPS executor ${EXECUTOR_TEST_OP})
+endif()
+
cc_library(tensor_array SRCS tensor_array.cc DEPS lod_tensor)
cc_test(tensor_array_test SRCS tensor_array_test.cc DEPS tensor_array place)
diff --git a/paddle/framework/backward.cc b/paddle/framework/backward.cc
index 0a4688db9c..c966f97c2d 100644
--- a/paddle/framework/backward.cc
+++ b/paddle/framework/backward.cc
@@ -28,14 +28,15 @@ namespace paddle {
namespace framework {
static inline std::unique_ptr<OperatorBase> CreateGradOp(
- const OperatorBase& op) {
+ const OperatorBase& op,
+ const std::unordered_set<std::string>& no_grad_set) {
OpDescBind op_desc;
op_desc.SetInputMap(op.Inputs());
op_desc.SetOutputMap(op.Outputs());
op_desc.SetType(op.Type());
op_desc.SetAttrMap(op.Attrs());
auto& info = OpInfoMap::Instance().Get(op.Type());
- auto grad_descs = info.GradOpMaker()(op_desc);
+ auto grad_descs = info.GradOpMaker()(op_desc, no_grad_set);
std::vector<std::unique_ptr<OperatorBase>> grad_ops;
grad_ops.reserve(grad_descs.size());
std::transform(grad_descs.begin(), grad_descs.end(),
@@ -172,30 +173,14 @@ static std::unique_ptr<OperatorBase> BackwardRecursive(
std::to_string(i));
net->ops_[op_offset]->Rename(name, dup_outputs.back());
}
- // collect all the offset to append `add` op for each alias
- //
- // one variable is shared between multiple operators.
- // insert add operator one by one, then add it to output
- for (size_t output_idx = 0; output_idx < dup_outputs.size() - 1;
- ++output_idx) {
- auto insert_add_x = dup_outputs[output_idx];
- auto insert_add_y = dup_outputs[output_idx + 1];
- auto insert_add_out = name + "@SHARED@" + std::to_string(output_idx);
- // first add op inserted
- if (output_idx == dup_outputs.size() - 2) {
- insert_add_out = name;
- }
- if (output_idx != 0) {
- insert_add_y = name + "@SHARED@" + std::to_string(output_idx - 1);
- }
- insert_position.push_back(
- {dup_op.back(),
- OpRegistry::CreateOp("sum", {{"X", {insert_add_x, insert_add_y}}},
- {{"Out", {insert_add_out}}}, {})});
- }
+ // collect all the offset for each alias,
+ // insert a sum operator to add all aliases to output
+ insert_position.push_back(
+ {dup_op.back(), OpRegistry::CreateOp("sum", {{"X", dup_outputs}},
+ {{"Out", {name}}}, {})});
}
- // make sure the inserted `add` ops follow the BFS order.
+ // make sure the inserted `sum` ops follow the BFS order.
insert_position.sort(
[](const Pos& l, const Pos& r) { return l.first > r.first; });
@@ -203,7 +188,8 @@ static std::unique_ptr<OperatorBase> BackwardRecursive(
net->InsertOp(pos.first + 1, std::move(pos.second));
}
} else {
- std::unique_ptr<OperatorBase> grad_op(CreateGradOp(forwardOp));
+ std::unique_ptr<OperatorBase> grad_op(
+ CreateGradOp(forwardOp, no_grad_names));
ForEachVarName(grad_op->Inputs(), [&no_grad_names, &net, &grad_op](
const std::string& grad_input) {
@@ -288,7 +274,7 @@ std::vector<std::unique_ptr<OpDescBind>> MakeOpGrad(
const std::unique_ptr<OpDescBind>& op_desc,
std::unordered_set<std::string>& no_grad_vars) {
std::vector<std::unique_ptr<OpDescBind>> grad_op_descs;
- // All input gradients of forwarding operator do not need to calculat.
+ // All input gradients of forwarding operator do not need to calculate.
const std::vector<std::string>& inputs = op_desc->InputArgumentNames();
if (AllGradInSet(inputs, no_grad_vars)) {
return grad_op_descs; // empty vector
@@ -302,7 +288,9 @@ std::vector<std::unique_ptr<OpDescBind>> MakeOpGrad(
return grad_op_descs; // empty vector
}
- grad_op_descs = OpRegistry::CreateGradOpDescs(op_desc.get());
+ grad_op_descs = OpInfoMap::Instance()
+ .Get(op_desc->Type())
+ .GradOpMaker()(*op_desc, no_grad_vars);
std::list<std::unique_ptr<OpDescBind>> pending_fill_zeros_ops;
for (auto& desc : grad_op_descs) {
@@ -317,11 +305,6 @@ std::vector<std::unique_ptr<OpDescBind>> MakeOpGrad(
pending_fill_zeros_ops.push_back(std::move(fill_zeros_op));
}
}
- for (const std::string& out_name : desc->OutputArgumentNames()) {
- if (no_grad_vars.count(out_name)) {
- desc->Rename(out_name, kEmptyVarName);
- }
- }
}
for (auto& p : pending_fill_zeros_ops) {
diff --git a/paddle/framework/backward.h b/paddle/framework/backward.h
index 7ffe4c2810..f1ab805645 100644
--- a/paddle/framework/backward.h
+++ b/paddle/framework/backward.h
@@ -27,6 +27,8 @@ extern std::unique_ptr<OperatorBase> Backward(
const OperatorBase& forwardOp,
const std::unordered_set<std::string>& no_grad_vars);
+// TODO(jiayi): Add target as parameter and generate backward op
+// according to target.
void AppendBackward(ProgramDescBind& program_desc,
const std::unordered_set<std::string>& no_grad_vars);
diff --git a/paddle/framework/backward_test.cc b/paddle/framework/backward_test.cc
index 3b7cbcd989..9b15331dff 100644
--- a/paddle/framework/backward_test.cc
+++ b/paddle/framework/backward_test.cc
@@ -169,6 +169,45 @@ class MultInOutOpMaker : public OpProtoAndCheckerMaker {
}
};
+class MinusGradOpDescMaker : public GradOpDescMakerBase {
+ public:
+ using GradOpDescMakerBase::GradOpDescMakerBase;
+
+ std::vector<std::unique_ptr<OpDescBind>> operator()() const override {
+ std::vector<std::unique_ptr<OpDescBind>> retv;
+ auto x_g = InputGrad("X");
+ if (!x_g.empty()) {
+ auto *op_desc = new OpDescBind();
+ op_desc->SetType("scale");
+ op_desc->SetInput("X", OutputGrad("Out"));
+ op_desc->SetOutput("Out", x_g);
+ op_desc->SetAttr("scale", 1.0f);
+ retv.emplace_back(op_desc);
+ }
+
+ auto y_g = InputGrad("Y");
+ if (!y_g.empty()) {
+ auto *op_desc = new OpDescBind();
+ op_desc->SetType("scale");
+ op_desc->SetInput("X", OutputGrad("Out"));
+ op_desc->SetOutput("Out", y_g);
+ op_desc->SetAttr("scale", -1.0f);
+ retv.emplace_back(op_desc);
+ }
+ return retv;
+ }
+};
+
+class MinusOpMaker : public OpProtoAndCheckerMaker {
+ public:
+ MinusOpMaker(OpProto *proto, OpAttrChecker *op_checker)
+ : OpProtoAndCheckerMaker(proto, op_checker) {
+ AddInput("X", "");
+ AddInput("Y", "");
+ AddOutput("Out", "");
+ AddComment("minus for unittest");
+ }
+};
} // namespace framework
} // namespace paddle
@@ -187,6 +226,7 @@ REGISTER_OP_WITHOUT_GRADIENT(fc, f::FcOp, f::FcOpMaker);
REGISTER_OP(many_output_op, f::NOP, f::ManyOutputOpMaker, many_output_op_grad,
f::NOP);
REGISTER_OP(mult_in_out, f::NOP, f::MultInOutOpMaker, mult_in_out_grad, f::NOP);
+REGISTER_OPERATOR(minus, f::NOP, f::MinusOpMaker, f::MinusGradOpDescMaker);
TEST(Backward, simple_op_not_need_grad) {
auto fwd = f::OpRegistry::CreateOp(
@@ -395,12 +435,13 @@ TEST(Backward, linear_net_intermediate_variable_has_no_grad) {
2UL /* external input number */
+ 1UL /* external output number*/
+ 1UL /* number of gradient of external output*/
- + 2U /* internal variable number*/);
+ + 2UL /* internal variable number*/
+ );
EXPECT_EQ(grad_fc.Outputs(all).size(),
2UL /* input number of mul*/
- + 2UL /* input number of rowwise_add
- */
- + 1UL /* input number of sigmod */);
+ + 2UL /* input number of rowwise_add*/
+ + 1UL /* input number of sigmod */
+ - 1UL /* out2 is not needed*/);
EXPECT_EQ(bwd_net->ops_[1]->Inputs(all).size(), 0UL);
EXPECT_EQ(bwd_net->ops_[1]->Outputs(all).size(), 0UL);
EXPECT_EQ(bwd_net->ops_[2]->Inputs(all).size(), 0UL);
@@ -451,6 +492,7 @@ TEST(Backward, default_attribute) {
op->SetInput("X", {"x"});
op->SetInput("Y", {"y"});
op->SetOutput("Out", {"out"});
+ op->CheckAttrs();
AppendBackward(program, {});
@@ -579,8 +621,7 @@ TEST(Backward, intermedia_var_no_grad) {
std::vector<std::string>({f::GradVarName("out4")}));
EXPECT_EQ(grad_op4->Output(f::GradVarName("X")),
std::vector<std::string>({f::GradVarName("out1")}));
- EXPECT_EQ(grad_op4->Output(f::GradVarName("Y")),
- std::vector<std::string>({f::kEmptyVarName}));
+ EXPECT_EQ(grad_op4->Output(f::GradVarName("Y")), std::vector<std::string>());
}
TEST(Backward, var_no_grad) {
@@ -618,8 +659,7 @@ TEST(Backward, var_no_grad) {
std::vector<std::string>({f::GradVarName("z2")}));
EXPECT_EQ(grad_op2->Output(f::GradVarName("X")),
std::vector<std::string>({f::GradVarName("y1")}));
- EXPECT_EQ(grad_op2->Output(f::GradVarName("H")),
- std::vector<std::string>({f::kEmptyVarName}));
+ EXPECT_EQ(grad_op2->Output(f::GradVarName("H")), std::vector<std::string>());
f::OpDescBind *fill_zero_op = block->AllOps()[3];
ASSERT_EQ(fill_zero_op->Type(), "fill_zeros_like");
@@ -717,4 +757,19 @@ TEST(Backward, shared_var) {
std::vector<std::string>({f::GradVarName("x1")}));
EXPECT_EQ(grad_op1->Output(f::GradVarName("b")),
std::vector<std::string>({f::GradVarName("b1")}));
+}
+
+TEST(Backward, half_backward) {
+ f::ProgramDesc *program_desc = GetNewProgramDesc();
+ f::ProgramDescBind &program = f::ProgramDescBind::Instance(program_desc);
+ f::BlockDescBind *block = program.Block(0);
+ auto *op1 = block->AppendOp();
+ op1->SetType("minus");
+ op1->SetInput("X", {"a"});
+ op1->SetInput("Y", {"b"});
+ op1->SetOutput("Out", {"out"});
+
+ AppendBackward(program, {"b"});
+ auto ops = block->AllOps();
+ ASSERT_EQ(2UL, ops.size());
}
\ No newline at end of file
diff --git a/paddle/framework/block_desc.cc b/paddle/framework/block_desc.cc
index 509aa235d3..b77d5525d4 100644
--- a/paddle/framework/block_desc.cc
+++ b/paddle/framework/block_desc.cc
@@ -91,9 +91,5 @@ BlockDescBind *BlockDescBind::ParentBlock() const {
return prog_->Block(static_cast<size_t>(this->desc_->parent_idx()));
}
-void OpDescBind::SetBlockAttr(const std::string &name, BlockDescBind &block) {
- BlockDesc *desc = block.RawPtr();
- this->attrs_[name] = desc;
-}
} // namespace framework
} // namespace paddle
diff --git a/paddle/framework/details/op_registry.h b/paddle/framework/details/op_registry.h
index daa474e8c5..ca8584b78a 100644
--- a/paddle/framework/details/op_registry.h
+++ b/paddle/framework/details/op_registry.h
@@ -97,8 +97,10 @@ struct OpInfoFiller<T, kOpProtoAndCheckerMaker> {
template <typename T>
struct OpInfoFiller<T, kGradOpDescMaker> {
void operator()(const char* op_type, OpInfo* info) const {
- info->grad_op_maker_ = [](const OpDescBind& fwd_op) {
- T maker(fwd_op);
+ info->grad_op_maker_ = [](
+ const OpDescBind& fwd_op,
+ const std::unordered_set<std::string>& no_grad_set) {
+ T maker(fwd_op, no_grad_set);
return maker();
};
}
diff --git a/paddle/framework/executor.cc b/paddle/framework/executor.cc
new file mode 100644
index 0000000000..c388b2198e
--- /dev/null
+++ b/paddle/framework/executor.cc
@@ -0,0 +1,163 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/framework/executor.h"
+
+#include <algorithm>
+#include <iostream>
+#include <memory>
+#include <set>
+#include <vector>
+
+#include "paddle/framework/lod_tensor.h"
+#include "paddle/framework/op_registry.h"
+#include "paddle/framework/scope.h"
+
+namespace paddle {
+namespace framework {
+
+const std::string kFeedOpType = "feed";
+const std::string kFetchOpType = "fetch";
+
+Executor::Executor(const std::vector<platform::Place>& places) {
+ PADDLE_ENFORCE_GT(places.size(), 0);
+ device_contexts_.resize(places.size());
+ for (size_t i = 0; i < places.size(); i++) {
+ if (platform::is_cpu_place(places[i])) {
+ device_contexts_[i] = new platform::CPUDeviceContext(
+ boost::get<platform::CPUPlace>(places[i]));
+ } else if (platform::is_gpu_place(places[i])) {
+#ifdef PADDLE_WITH_CUDA
+ device_contexts_[i] = new platform::CUDADeviceContext(
+ boost::get<platform::GPUPlace>(places[i]));
+#else
+ PADDLE_THROW(
+ "'GPUPlace' is not supported, Please re-compile with WITH_GPU "
+ "option");
+#endif
+ }
+ }
+}
+
+Executor::~Executor() {
+ for (auto& device_context : device_contexts_) {
+ delete device_context;
+ }
+}
+
+void Executor::Run(const ProgramDesc& pdesc, Scope* scope, int block_id) {
+ // TODO(tonyyang-svail):
+ // - only runs on the first device (i.e. no interdevice communication)
+ // - will change to use multiple blocks for RNN op and Cond Op
+ PADDLE_ENFORCE_GT(pdesc.blocks_size(), block_id);
+ auto& block = pdesc.blocks(block_id);
+ auto& device = device_contexts_[0];
+
+ // Instantiate all the vars in the global scope
+ for (auto& var : block.vars()) {
+ scope->NewVar(var.name());
+ }
+
+ Scope& local_scope = scope->NewScope();
+
+ std::vector<bool> should_run = Prune(pdesc, block_id);
+ PADDLE_ENFORCE_EQ(should_run.size(), static_cast<size_t>(block.ops_size()));
+ for (size_t i = 0; i < should_run.size(); ++i) {
+ if (should_run[i]) {
+ for (auto& var : block.ops(i).outputs()) {
+ for (auto& argu : var.arguments()) {
+ if (local_scope.FindVar(argu) == nullptr) {
+ local_scope.NewVar(argu);
+ }
+ }
+ }
+ auto op = paddle::framework::OpRegistry::CreateOp(block.ops(i));
+ op->Run(local_scope, *device);
+ }
+ }
+
+ // TODO(tonyyang-svail):
+ // - Destroy local_scope
+}
+
+std::vector<bool> Prune(const ProgramDesc& pdesc, int block_id) {
+ // TODO(tonyyang-svail):
+ // - will change to use multiple blocks for RNN op and Cond Op
+
+ auto& block = pdesc.blocks(block_id);
+ auto& ops = block.ops();
+
+ bool expect_feed = true;
+ for (auto& op_desc : ops) {
+ PADDLE_ENFORCE(op_desc.type() != kFeedOpType || expect_feed,
+ "All FeedOps are at the beginning of the ProgramDesc");
+ expect_feed = (op_desc.type() == kFeedOpType);
+ }
+
+ bool expect_fetch = true;
+ for (auto op_iter = ops.rbegin(); op_iter != ops.rend(); ++op_iter) {
+ auto& op_desc = *op_iter;
+ PADDLE_ENFORCE(op_desc.type() != kFetchOpType || expect_fetch,
+ "All FetchOps must at the end of the ProgramDesc");
+ expect_fetch = (op_desc.type() == kFetchOpType);
+ }
+
+ std::set<std::string> dependent_vars;
+ std::vector<bool> should_run;
+ for (auto op_iter = ops.rbegin(); op_iter != ops.rend(); ++op_iter) {
+ auto& op_desc = *op_iter;
+
+ bool found_dependent_vars = false;
+ for (auto& var : op_desc.outputs()) {
+ for (auto& argu : var.arguments()) {
+ if (dependent_vars.count(argu) != 0) {
+ found_dependent_vars = true;
+ }
+ }
+ }
+
+ if (op_desc.type() == kFetchOpType || found_dependent_vars) {
+ // erase its output to the dependency graph
+ for (auto& var : op_desc.outputs()) {
+ for (auto& argu : var.arguments()) {
+ dependent_vars.erase(argu);
+ }
+ }
+
+ // insert its input to the dependency graph
+ for (auto& var : op_desc.inputs()) {
+ for (auto& argu : var.arguments()) {
+ dependent_vars.insert(argu);
+ }
+ }
+
+ should_run.push_back(true);
+ } else {
+ should_run.push_back(false);
+ }
+ }
+
+ // TODO(tonyyang-svail):
+ // - check this after integration of Init
+ // PADDLE_ENFORCE(dependent_vars.empty());
+
+ // since we are traversing the ProgramDesc in reverse order
+ // we reverse the should_run vector
+ std::reverse(should_run.begin(), should_run.end());
+
+ return should_run;
+}
+
+} // namespace framework
+} // namespace paddle
diff --git a/paddle/framework/executor.h b/paddle/framework/executor.h
new file mode 100644
index 0000000000..4e3bc2c0a5
--- /dev/null
+++ b/paddle/framework/executor.h
@@ -0,0 +1,55 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+
+#include "paddle/framework/framework.pb.h"
+#include "paddle/framework/op_info.h"
+#include "paddle/framework/scope.h"
+#include "paddle/framework/tensor.h"
+
+namespace paddle {
+namespace framework {
+
+class Executor {
+ public:
+ explicit Executor(const std::vector<platform::Place>& places);
+ ~Executor();
+
+ /* @Brief
+ * Runtime evaluation of the given ProgramDesc under certain Scope
+ *
+ * @param
+ * ProgramDesc
+ * Scope
+ */
+ void Run(const ProgramDesc&, Scope*, int);
+
+ private:
+ std::vector<platform::DeviceContext*> device_contexts_;
+};
+
+/* @Brief
+ * Pruning the graph
+ *
+ * @param
+ * ProgramDesc
+ *
+ * @return
+ * vector<bool> Same size as ops. Indicates whether an op should be run.
+ */
+std::vector<bool> Prune(const ProgramDesc& pdesc, int block_id);
+
+} // namespace framework
+} // namespace paddle
diff --git a/paddle/framework/executor_test.cc b/paddle/framework/executor_test.cc
new file mode 100644
index 0000000000..8c7db3b290
--- /dev/null
+++ b/paddle/framework/executor_test.cc
@@ -0,0 +1,329 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/framework/executor.h"
+
+#include <memory>
+#include <vector>
+
+#include "gflags/gflags.h"
+#include "gtest/gtest.h"
+#include "paddle/framework/attribute.h"
+#include "paddle/framework/backward.h"
+#include "paddle/framework/block_desc.h"
+#include "paddle/framework/op_desc.h"
+#include "paddle/framework/op_registry.h"
+#include "paddle/framework/operator.h"
+
+USE_OP(elementwise_add);
+USE_OP(gaussian_random);
+USE_OP(feed);
+USE_OP(fetch);
+USE_OP(mul);
+USE_OP(sum);
+USE_OP(squared_l2_distance);
+USE_OP(fill_constant);
+USE_OP(sgd);
+
+using namespace paddle::platform;
+using namespace paddle::framework;
+
+void AddOp(const std::string& type, const VariableNameMap& inputs,
+ const VariableNameMap& outputs, AttributeMap attrs,
+ paddle::framework::BlockDescBind* block) {
+ // insert output
+ for (auto kv : outputs) {
+ for (auto v : kv.second) {
+ auto var = block->NewVar(v);
+ var->SetDataType(paddle::framework::DataType::FP32);
+ }
+ }
+
+ // insert op
+ auto op = block->AppendOp();
+ op->SetType(type);
+ for (auto& kv : inputs) {
+ op->SetInput(kv.first, kv.second);
+ }
+ for (auto& kv : outputs) {
+ op->SetOutput(kv.first, kv.second);
+ }
+ op->SetAttrMap(attrs);
+ op->CheckAttrs();
+}
+
+// Tensors in feed value variable will only be in CPUPlace
+// So we can memcpy the data from vector<T> to feed_value
+template <typename T>
+void SetFeedVariable(const std::vector<std::vector<T>>& inputs,
+ const std::vector<std::vector<int64_t>>& dims) {
+ Variable* g_feed_value = GetGlobalScope().FindVar("feed_value");
+ auto& feed_inputs =
+ *(g_feed_value->GetMutable<std::vector<paddle::framework::Tensor>>());
+ size_t size = inputs.size();
+ feed_inputs.resize(size);
+ for (size_t i = 0; i < size; i++) {
+ T* dst = feed_inputs[i].mutable_data<T>(make_ddim(dims[i]), CPUPlace());
+ memcpy(dst, inputs[i].data(), inputs[i].size() * sizeof(T));
+ }
+}
+
+// Tensors in fetch value variable will only be in CPUPlace
+// So we can memcpy the data from fetch_value to vector<T>
+template <typename T>
+std::vector<std::vector<T>> GetFetchVariable() {
+ Variable* g_fetch_value = GetGlobalScope().FindVar("fetch_value");
+ auto& fetch_outputs =
+ *(g_fetch_value->GetMutable<std::vector<paddle::framework::Tensor>>());
+
+ size_t size = fetch_outputs.size();
+ std::vector<std::vector<T>> result;
+ result.reserve(size);
+ for (size_t i = 0; i < size; i++) {
+ std::vector<T> tmp;
+ tmp.resize(fetch_outputs[i].numel());
+ memcpy(tmp.data(), fetch_outputs[i].data<T>(),
+ fetch_outputs[i].numel() * sizeof(T));
+ result.push_back(tmp);
+ }
+
+ return result;
+}
+
+class ExecutorTesterRandom : public ::testing::Test {
+ public:
+ virtual void SetUp() override {
+ int input_dim = 3, batch_size = 2, embed_dim = 5;
+
+ auto temp_init_root_block = init_pdesc_.add_blocks();
+ temp_init_root_block->set_idx(0);
+ temp_init_root_block->set_parent_idx(-1);
+ paddle::framework::ProgramDescBind& init_program =
+ paddle::framework::ProgramDescBind::Instance(&init_pdesc_);
+ paddle::framework::BlockDescBind* init_root_block = init_program.Block(0);
+
+ AddOp("gaussian_random", {}, {{"Out", {"w1"}}},
+ {{"dims", std::vector<int>{input_dim, embed_dim}}}, init_root_block);
+ AddOp("gaussian_random", {}, {{"Out", {"w2"}}},
+ {{"dims", std::vector<int>{embed_dim, input_dim}}}, init_root_block);
+ AddOp("fetch", {{"Input", {"w1"}}}, {}, {{"col", 0}}, init_root_block);
+ AddOp("fetch", {{"Input", {"w2"}}}, {}, {{"col", 1}}, init_root_block);
+
+ // flush
+ init_program.Proto();
+
+ // run block
+ auto temp_root_block = pdesc_.add_blocks();
+ temp_root_block->set_idx(0);
+ temp_root_block->set_parent_idx(-1);
+ paddle::framework::ProgramDescBind& program =
+ paddle::framework::ProgramDescBind::Instance(&pdesc_);
+ paddle::framework::BlockDescBind* root_block = program.Block(0);
+
+ // feed data
+ inputs_.push_back({1.0, 1.0, 1.0, 1.0, 1.0, 1.0});
+ dims_.push_back({batch_size, input_dim});
+ AddOp("feed", {}, {{"Out", {"a"}}},
+ {{"dims", std::vector<int>{batch_size, input_dim}}, {"col", 0}},
+ root_block);
+
+ // forward
+ AddOp("mul", {{"X", {"a"}}, {"Y", {"w1"}}}, {{"Out", {"b"}}}, {},
+ root_block);
+ AddOp("mul", {{"X", {"b"}}, {"Y", {"w2"}}}, {{"Out", {"a_out"}}}, {},
+ root_block);
+ AddOp("squared_l2_distance", {{"X", {"a"}}, {"Y", {"a_out"}}},
+ {{"Out", {"l2_distance"}}, {"sub_result", {"l2_distance_sub"}}}, {},
+ root_block);
+
+ // backward
+ AddOp("fill_constant", {}, {{"Out", {"l2_distance@GRAD"}}},
+ {{"shape", std::vector<int>{batch_size, 1}}, {"value", float(1.0)}},
+ root_block);
+ AppendBackward(program, {});
+
+ // update
+ AddOp("fill_constant", {}, {{"Out", {"learning_rate"}}},
+ {{"shape", std::vector<int>{1}}, {"value", float(0.001)}},
+ root_block);
+ AddOp("sgd", {{"Param", {"w1"}},
+ {"LearningRate", {"learning_rate"}},
+ {"Grad", {"w1@GRAD"}}},
+ {{"ParamOut", {"w1"}}}, {}, root_block);
+ AddOp("sgd", {{"Param", {"w2"}},
+ {"LearningRate", {"learning_rate"}},
+ {"Grad", {"w2@GRAD"}}},
+ {{"ParamOut", {"w2"}}}, {}, root_block);
+
+ AddOp("fetch", {{"Input", {"w1"}}}, {}, {{"col", 0}}, root_block);
+ AddOp("fetch", {{"Input", {"w2"}}}, {}, {{"col", 1}}, root_block);
+ AddOp("fetch", {{"Input", {"l2_distance"}}}, {}, {{"col", 0}}, root_block);
+
+ // flush
+ program.Proto();
+ }
+
+ protected:
+ ProgramDesc init_pdesc_;
+ ProgramDesc pdesc_;
+ std::vector<std::vector<float>> inputs_;
+ std::vector<std::vector<int64_t>> dims_;
+};
+
+class ExecutorTesterFeedAndFetch : public ::testing::Test {
+ public:
+ virtual void SetUp() override {
+ auto temp_root_block = pdesc_.add_blocks();
+ temp_root_block->set_idx(0);
+ temp_root_block->set_parent_idx(-1);
+
+ // wrap to BlockDescBind
+ paddle::framework::ProgramDescBind& program =
+ paddle::framework::ProgramDescBind::Instance(&pdesc_);
+ paddle::framework::BlockDescBind* root_block = program.Block(0);
+
+ std::vector<int> dim{6};
+
+ AddOp("feed", {}, {{"Out", {"a"}}}, {{"dims", dim}, {"col", 0}},
+ root_block);
+ AddOp("feed", {}, {{"Out", {"b"}}}, {{"dims", dim}, {"col", 1}},
+ root_block);
+ AddOp("fetch", {{"Input", {"a"}}}, {}, {{"col", 0}}, root_block);
+ AddOp("fetch", {{"Input", {"b"}}}, {}, {{"col", 1}}, root_block);
+
+ // flush
+ program.Proto();
+
+ std::vector<float> vec1 = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0};
+ std::vector<float> vec2 = {4.0, 5.0, 6.0, 7.0, 8.0, 9.0};
+ inputs_.push_back(vec1);
+ inputs_.push_back(vec2);
+ dims_.push_back({static_cast<int64_t>(vec1.size())});
+ dims_.push_back({static_cast<int64_t>(vec2.size())});
+ }
+
+ protected:
+ ProgramDesc pdesc_;
+ std::vector<std::vector<float>> inputs_;
+ std::vector<std::vector<int64_t>> dims_;
+};
+
+#ifndef PADDLE_WITH_CUDA
+TEST_F(ExecutorTesterRandom, CPU) {
+ std::vector<Place> places;
+ CPUPlace cpu_place;
+ places.push_back(cpu_place);
+
+ // We have a global Scope and BuddyAllocator, and we must ensure
+ // global BuddyAllocator is initialized before global Scope. Thus,
+ // global Scope will deconstruct before BuddyAllocator. Otherwise,
+ // "pointer being freed was not allocated" error will appear.
+ paddle::memory::Used(cpu_place);
+
+ std::unique_ptr<Executor> executor(new Executor(places));
+
+ executor->Run(init_pdesc_, &GetGlobalScope(), 0);
+ SetFeedVariable<float>(inputs_, dims_);
+ executor->Run(pdesc_, &GetGlobalScope(), 0);
+ std::vector<std::vector<float>> result = GetFetchVariable<float>();
+}
+
+TEST_F(ExecutorTesterFeedAndFetch, CPU) {
+ std::vector<Place> places;
+ CPUPlace cpu_place;
+ places.push_back(cpu_place);
+
+ // We have a global Scope and BuddyAllocator, and we must ensure
+ // global BuddyAllocator is initialized before global Scope. Thus,
+ // global Scope will deconstruct before BuddyAllocator. Otherwise,
+ // "pointer being freed was not allocated" error will appear.
+ paddle::memory::Used(cpu_place);
+
+ std::unique_ptr<Executor> executor(new Executor(places));
+
+ for (int batch_id = 0; batch_id < 3; batch_id++) {
+ SetFeedVariable<float>(inputs_, dims_);
+ executor->Run(pdesc_, &GetGlobalScope(), 0);
+ std::vector<std::vector<float>> result = GetFetchVariable<float>();
+ PADDLE_ENFORCE_EQ(result.size(), inputs_.size());
+ for (size_t i = 0; i < result.size(); ++i) {
+ PADDLE_ENFORCE_EQ(result[i].size(), inputs_[i].size());
+ for (size_t j = 0; j < result[i].size(); ++j) {
+ PADDLE_ENFORCE_EQ(result[i][j], inputs_[i][j]);
+ }
+ }
+ }
+}
+#else
+TEST_F(ExecutorTesterRandom, GPU) {
+ std::vector<Place> places;
+ GPUPlace gpu_place(0);
+ places.push_back(gpu_place);
+
+ // We have a global Scope and BuddyAllocator, and we must ensure
+ // global BuddyAllocator is initialized before global Scope. Thus,
+ // global Scope will deconstruct before BuddyAllocator. Otherwise,
+ // "pointer being freed was not allocated" error will appear.
+ // If paddle is compiled with GPU, both CPU and GPU BuddyAllocator
+ // need to be used at first.
+ paddle::memory::Used(CPUPlace());
+ paddle::memory::Used(gpu_place);
+
+ std::unique_ptr<Executor> executor(new Executor(places));
+
+ executor->Run(init_pdesc_, &GetGlobalScope(), 0);
+ for (int batch_id = 0; batch_id < 3; batch_id++) {
+ SetFeedVariable<float>(inputs_, dims_);
+ executor->Run(pdesc_, &GetGlobalScope(), 0);
+ }
+}
+
+TEST_F(ExecutorTesterFeedAndFetch, GPU) {
+ std::vector<Place> places;
+ GPUPlace gpu_place(0);
+ places.push_back(gpu_place);
+ // We have a global Scope and BuddyAllocator, and we must ensure
+ // global BuddyAllocator is initialized before global Scope. Thus,
+ // global Scope will deconstruct before BuddyAllocator. Otherwise,
+ // "pointer being freed was not allocated" error will appear.
+ // If paddle is compiled with GPU, both CPU and GPU BuddyAllocator
+ // need to be used at first.
+ paddle::memory::Used(CPUPlace());
+ paddle::memory::Used(gpu_place);
+
+ std::unique_ptr<Executor> executor(new Executor(places));
+
+ for (int batch_id = 0; batch_id < 3; batch_id++) {
+ SetFeedVariable<float>(inputs_, dims_);
+ executor->Run(pdesc_, &GetGlobalScope(), 0);
+ std::vector<std::vector<float>> result = GetFetchVariable<float>();
+ PADDLE_ENFORCE_EQ(result.size(), inputs_.size());
+ for (size_t i = 0; i < result.size(); ++i) {
+ PADDLE_ENFORCE_EQ(result[i].size(), inputs_[i].size());
+ for (size_t j = 0; j < result[i].size(); ++j) {
+ PADDLE_ENFORCE_EQ(result[i][j], inputs_[i][j]);
+ }
+ }
+ }
+}
+#endif
+
+DECLARE_double(fraction_of_gpu_memory_to_use);
+
+int main(int argc, char** argv) {
+ testing::InitGoogleTest(&argc, argv);
+ // Use less GPU memory for unittest.
+ FLAGS_fraction_of_gpu_memory_to_use = 0.25;
+ return RUN_ALL_TESTS();
+}
\ No newline at end of file
diff --git a/paddle/framework/grad_op_desc_maker.h b/paddle/framework/grad_op_desc_maker.h
index e9ae6e2206..d7366b11ec 100644
--- a/paddle/framework/grad_op_desc_maker.h
+++ b/paddle/framework/grad_op_desc_maker.h
@@ -13,6 +13,8 @@
limitations under the License. */
#pragma once
+#include <string>
+#include <unordered_set>
#include "paddle/framework/op_desc.h"
#include "paddle/framework/operator.h"
@@ -21,27 +23,44 @@ namespace framework {
class GradOpDescMakerBase {
public:
- explicit GradOpDescMakerBase(const OpDescBind& fwd_op) : fwd_op_(fwd_op) {}
+ explicit GradOpDescMakerBase(
+ const OpDescBind& fwd_op,
+ const std::unordered_set<std::string>& no_grad_set)
+ : fwd_op_(fwd_op), no_grad_set_(no_grad_set) {}
virtual ~GradOpDescMakerBase() = default;
virtual std::vector<std::unique_ptr<OpDescBind>> operator()() const = 0;
protected:
- static std::vector<std::string> ToGradNames(
- const std::vector<std::string>& var_names) {
+ std::vector<std::string> InputGrad(const std::string& name,
+ bool drop_empty_grad = true) const {
std::vector<std::string> ret_val;
+ auto var_names = this->Input(name);
ret_val.reserve(var_names.size());
- std::transform(var_names.begin(), var_names.end(),
- std::back_inserter(ret_val), GradVarName);
- return ret_val;
- }
-
- std::vector<std::string> InputGrad(const std::string& name) const {
- return ToGradNames(fwd_op_.Input(name));
+ std::transform(
+ var_names.begin(), var_names.end(), std::back_inserter(ret_val),
+ [this](const std::string& fwd_var_name) -> std::string {
+ auto g_name = GradVarName(fwd_var_name);
+ return no_grad_set_.count(g_name) == 0 ? g_name : kEmptyVarName;
+ });
+ if (!drop_empty_grad) {
+ return ret_val;
+ }
+ std::vector<std::string> dropped_ret_val;
+ dropped_ret_val.reserve(ret_val.size());
+ std::copy_if(ret_val.begin(), ret_val.end(),
+ std::back_inserter(dropped_ret_val),
+ [](const std::string& str) { return str != kEmptyVarName; });
+ return dropped_ret_val;
}
std::vector<std::string> OutputGrad(const std::string& name) const {
- return ToGradNames(fwd_op_.Output(name));
+ std::vector<std::string> ret_val;
+ auto onames = this->Output(name);
+ ret_val.reserve(onames.size());
+ std::transform(onames.begin(), onames.end(), std::back_inserter(ret_val),
+ GradVarName);
+ return ret_val;
}
std::vector<std::string> InputNames() const {
@@ -75,6 +94,7 @@ class GradOpDescMakerBase {
private:
const OpDescBind& fwd_op_;
+ const std::unordered_set<std::string>& no_grad_set_;
};
class SingleGradOpDescMaker : public GradOpDescMakerBase {
@@ -91,6 +111,7 @@ class SingleGradOpDescMaker : public GradOpDescMakerBase {
virtual std::unique_ptr<OpDescBind> Apply() const = 0;
};
+template <bool DropEmptyIG = true>
class DefaultGradOpDescMaker : public SingleGradOpDescMaker {
public:
using SingleGradOpDescMaker::SingleGradOpDescMaker;
@@ -102,7 +123,8 @@ class DefaultGradOpDescMaker : public SingleGradOpDescMaker {
for (auto& input_param : this->InputNames()) {
grad->SetInput(input_param, this->Input(input_param));
- grad->SetOutput(GradVarName(input_param), this->InputGrad(input_param));
+ grad->SetOutput(GradVarName(input_param),
+ this->InputGrad(input_param, DropEmptyIG));
}
for (auto& output_param : this->OutputNames()) {
diff --git a/paddle/framework/op_desc.cc b/paddle/framework/op_desc.cc
index e7538b4af3..a5d515bbca 100644
--- a/paddle/framework/op_desc.cc
+++ b/paddle/framework/op_desc.cc
@@ -100,6 +100,12 @@ void OpDescBind::SetAttr(const std::string &name, const Attribute &v) {
need_update_ = true;
}
+void OpDescBind::SetBlockAttr(const std::string &name, BlockDescBind &block) {
+ BlockDesc *desc = block.RawPtr();
+ this->attrs_[name] = desc;
+ need_update_ = true;
+}
+
void OpDescBind::SetAttrMap(
const std::unordered_map<std::string, Attribute> &attr_map) {
attrs_ = attr_map;
@@ -211,6 +217,15 @@ static InferShapeFuncMap &InferShapeFuncs() {
return *g_map;
}
+void OpDescBind::CheckAttrs() {
+ PADDLE_ENFORCE(!Type().empty(),
+ "CheckAttr() can not be called before type is setted.");
+ const auto *checker = OpInfoMap::Instance().Get(Type()).Checker();
+ PADDLE_ENFORCE_NOT_NULL(checker, "Operator \"%s\" has no registered checker.",
+ Type());
+ checker->Check(attrs_);
+}
+
void OpDescBind::InferShape(const BlockDescBind &block) const {
auto &funcs = InferShapeFuncs();
auto it = funcs.find(this->Type());
diff --git a/paddle/framework/op_desc.h b/paddle/framework/op_desc.h
index 81c4225041..90155fadea 100644
--- a/paddle/framework/op_desc.h
+++ b/paddle/framework/op_desc.h
@@ -100,6 +100,8 @@ class OpDescBind {
return &this->attrs_;
}
+ void CheckAttrs();
+
void InferShape(const BlockDescBind &block) const;
private:
diff --git a/paddle/framework/op_registry.cc b/paddle/framework/op_registry.cc
index b118edae17..504afbd5db 100644
--- a/paddle/framework/op_registry.cc
+++ b/paddle/framework/op_registry.cc
@@ -59,16 +59,5 @@ std::unique_ptr<OperatorBase> OpRegistry::CreateOp(const OpDescBind& op_desc) {
op_desc.GetAttrMap());
}
-std::vector<std::unique_ptr<OpDescBind>> OpRegistry::CreateGradOpDescs(
- OpDescBind* op_desc) {
- auto& info = OpInfoMap::Instance().Get(op_desc->Type());
-
- if (info.Checker() != nullptr) {
- info.Checker()->Check(*op_desc->MutableAttrMap());
- }
-
- return info.grad_op_maker_(*op_desc);
-}
-
} // namespace framework
} // namespace paddle
diff --git a/paddle/framework/op_registry.h b/paddle/framework/op_registry.h
index 5ca3af52a6..226e8ddcd4 100644
--- a/paddle/framework/op_registry.h
+++ b/paddle/framework/op_registry.h
@@ -79,9 +79,6 @@ class OpRegistry {
static std::unique_ptr<OperatorBase> CreateOp(const OpDesc& op_desc);
- static std::vector<std::unique_ptr<OpDescBind>> CreateGradOpDescs(
- OpDescBind* op_desc);
-
static std::unique_ptr<OperatorBase> CreateOp(const OpDescBind& op_desc);
};
@@ -160,17 +157,18 @@ class OpKernelRegistrar : public Registrar {
/**
* Macro to register Operator.
*/
-#define REGISTER_OP(op_type, op_class, op_maker_class, grad_op_type, \
- grad_op_class) \
- REGISTER_OPERATOR(grad_op_type, grad_op_class); \
- class _GradOpDescMaker_##grad_op_type##_ \
- : public ::paddle::framework::DefaultGradOpDescMaker { \
- using ::paddle::framework::DefaultGradOpDescMaker::DefaultGradOpDescMaker; \
- \
- protected: \
- virtual std::string GradOpType() const { return #grad_op_type; } \
- }; \
- REGISTER_OPERATOR(op_type, op_class, _GradOpDescMaker_##grad_op_type##_, \
+#define REGISTER_OP(op_type, op_class, op_maker_class, grad_op_type, \
+ grad_op_class) \
+ REGISTER_OPERATOR(grad_op_type, grad_op_class); \
+ class _GradOpDescMaker_##grad_op_type##_ \
+ : public ::paddle::framework::DefaultGradOpDescMaker<true> { \
+ using ::paddle::framework::DefaultGradOpDescMaker< \
+ true>::DefaultGradOpDescMaker; \
+ \
+ protected: \
+ virtual std::string GradOpType() const { return #grad_op_type; } \
+ }; \
+ REGISTER_OPERATOR(op_type, op_class, _GradOpDescMaker_##grad_op_type##_, \
op_maker_class);
#define REGISTER_OP_WITHOUT_GRADIENT(op_type, op_class, op_maker_class) \
diff --git a/paddle/framework/operator.h b/paddle/framework/operator.h
index 15f80b5720..97a142d5f1 100644
--- a/paddle/framework/operator.h
+++ b/paddle/framework/operator.h
@@ -289,6 +289,15 @@ class ExecutionContext {
return device_context_;
}
+#ifdef PADDLE_WITH_CUDA
+ const platform::CUDADeviceContext& cuda_device_context() const {
+ PADDLE_ENFORCE(platform::is_gpu_place(device_context_.GetPlace()));
+ auto cuda_ctx =
+ reinterpret_cast<const platform::CUDADeviceContext*>(&device_context_);
+ return *cuda_ctx;
+ }
+#endif
+
private:
const OperatorBase& op_;
const Scope& scope_;
diff --git a/paddle/framework/scope.cc b/paddle/framework/scope.cc
index 080b4ac621..5821bac928 100644
--- a/paddle/framework/scope.cc
+++ b/paddle/framework/scope.cc
@@ -13,6 +13,9 @@ See the License for the specific language governing permissions and
limitations under the License. */
#include "paddle/framework/scope.h"
+
+#include <memory> // for unique_ptr
+#include <mutex> // for call_once
#include "paddle/string/printf.h"
namespace paddle {
@@ -62,5 +65,17 @@ void Scope::DropKids() {
kids_.clear();
}
+std::once_flag feed_variable_flag;
+
+framework::Scope& GetGlobalScope() {
+ static std::unique_ptr<framework::Scope> g_scope{nullptr};
+ std::call_once(feed_variable_flag, [&]() {
+ g_scope.reset(new framework::Scope());
+ g_scope->NewVar("feed_value");
+ g_scope->NewVar("fetch_value");
+ });
+ return *(g_scope.get());
+}
+
} // namespace framework
} // namespace paddle
diff --git a/paddle/framework/scope.h b/paddle/framework/scope.h
index 7047f0d55e..a8cfb107c2 100644
--- a/paddle/framework/scope.h
+++ b/paddle/framework/scope.h
@@ -73,5 +73,7 @@ class Scope {
DISABLE_COPY_AND_ASSIGN(Scope);
};
+framework::Scope& GetGlobalScope();
+
} // namespace framework
} // namespace paddle
diff --git a/paddle/framework/shape_inference.h b/paddle/framework/shape_inference.h
index 64aab16ae5..b93f980cf6 100644
--- a/paddle/framework/shape_inference.h
+++ b/paddle/framework/shape_inference.h
@@ -19,9 +19,6 @@ limitations under the License. */
namespace paddle {
namespace framework {
-// TODO(longfei): Once after both CompileTimeInferShapeContext and
-// RuntimeInferShapeContext get merged, we can rename InferShapeContext into
-// InferShapeContext so to replace the current InferShapeContext.
class InferShapeContext {
public:
virtual ~InferShapeContext() {}
diff --git a/paddle/framework/tensor.h b/paddle/framework/tensor.h
index ba82127d9c..3304d857ae 100644
--- a/paddle/framework/tensor.h
+++ b/paddle/framework/tensor.h
@@ -87,26 +87,31 @@ class Tensor {
/**
* @brief Copy the content of external tensor to a new place.
*
- * @param[in] src The external tensor.
- * @param[in] ctx The device context contains place where to store.
+ * @param[in] src The external tensor.
+ * @param[in] dst_place The dst place.
+ * @param[in] ctx The device context contains device resources.
*
* @note CopyFrom supports CPU <-> GPU, GPU <-> GPU.
*/
+ // TODO(qijun): https://github.com/PaddlePaddle/Paddle/issues/4647
+ // Remove `CopyFrom` and `CopyFromVector` from Tensor interface
+ // and make them global functions
template <typename T>
- inline void CopyFrom(const Tensor& src, const platform::Place& dst_place);
+ inline void CopyFrom(const Tensor& src, const platform::Place& dst_place,
+ const platform::DeviceContext& ctx);
/**
* @brief Copy the content of an external vector to a tensor.
*
- * @param[in] src The external vector.
- * @param[in] ctx The device context contains place where to store.
+ * @param[in] src The external tensor.
+ * @param[in] ctx The device context contains device resources.
*
* * @note CopyFromVector assumes that the tensor has been resized
* before invoking.
*/
template <typename T>
inline void CopyFromVector(const std::vector<T>& src,
- const platform::Place& dst_place);
+ const platform::DeviceContext& ctx);
/**
* @brief Return the slice of the tensor.
diff --git a/paddle/framework/tensor_array.cc b/paddle/framework/tensor_array.cc
index 2728bce1c1..06459cbfd7 100644
--- a/paddle/framework/tensor_array.cc
+++ b/paddle/framework/tensor_array.cc
@@ -76,6 +76,17 @@ LoDTensor PackDynamicBatch(const std::vector<LoDTensor>& source,
const std::vector<DySeqMeta>& meta, const LoD& lod,
size_t level);
+std::vector<size_t> GenDyBatchIndice(const DySeqMetaBatch& meta, int batch_id) {
+ // collect indice need to copy to the batch
+ std::vector<size_t> indice;
+ for (const auto& seq : meta) {
+ size_t id = seq.begin + batch_id;
+ if (id >= seq.end) break;
+ indice.push_back(id);
+ }
+ return indice;
+}
+
} // namespace detail
const LoDTensor& TensorArray::Read(size_t index) const {
@@ -95,7 +106,8 @@ void TensorArray::Write(size_t index, const LoDTensor& value) {
values_[index].Resize(value.dims());
values_[index].mutable_data<value_type>(platform::CPUPlace());
- values_[index].CopyFrom<value_type>(value, platform::CPUPlace());
+ values_[index].CopyFrom<value_type>(value, platform::CPUPlace(),
+ platform::CPUDeviceContext());
}
void TensorArray::WriteShared(size_t index, const LoDTensor& value) {
@@ -112,8 +124,8 @@ LoDTensor TensorArray::Pack(size_t level, const std::vector<DySeqMeta>& meta,
return detail::PackDynamicBatch(values_, meta, lod, level);
}
-std::vector<DySeqMeta> TensorArray::Unpack(const LoDTensor& source, int level,
- bool length_desend) {
+DySeqMetaBatch TensorArray::Unpack(const LoDTensor& source, int level,
+ bool length_desend) {
detail::DynamicBatchUnpacker unpacker(source, level,
length_desend /*descend*/);
@@ -128,6 +140,7 @@ std::vector<DySeqMeta> TensorArray::Unpack(const LoDTensor& source, int level,
Write(batch_id, unpacker.GetBatch(batch_id));
}
+ PADDLE_ENFORCE(!unpacker.meta.empty());
return unpacker.meta;
}
@@ -151,7 +164,8 @@ LoDTensor TensorArray::Stack() const {
for (size_t idx = 0; idx < size(); idx++) {
result.Slice<value_type>(idx, idx + 1)
- .CopyFrom<value_type>(Read(idx), platform::CPUPlace());
+ .CopyFrom<value_type>(Read(idx), platform::CPUPlace(),
+ platform::CPUDeviceContext());
}
return result;
}
@@ -182,7 +196,8 @@ void TensorArray::Unstack(const LoDTensor& source, bool data_shared) const {
// copy
value.Resize(value_dims);
value.CopyFrom<value_type>(source.Slice<value_type>(elem, elem + 1),
- platform::CPUPlace());
+ platform::CPUPlace(),
+ platform::CPUDeviceContext());
}
}
}
@@ -215,13 +230,7 @@ LoDTensor DynamicBatchUnpacker::GetBatch(size_t index) {
PADDLE_ENFORCE(!meta.empty(), "should build meta first");
LoDTensor result;
- // collect indice need to copy to the batch
- std::vector<size_t> indice;
- for (const auto& seq : meta) {
- size_t id = seq.begin + index;
- if (id >= seq.end) break;
- indice.push_back(id);
- }
+ auto indice = detail::GenDyBatchIndice(meta, index);
PADDLE_ENFORCE(!indice.empty(), "invalid batch at %d", index);
// copy the indice of records in LoDTensor
@@ -234,9 +243,10 @@ LoDTensor DynamicBatchUnpacker::GetBatch(size_t index) {
for (size_t i = 0; i < indice.size(); i++) {
auto index = indice[i];
auto target = result.Slice<value_type>(i, i + 1);
- auto source_ = source->Slice<value_type>(index, index + 1);
+ auto slice = source->Slice<value_type>(index, index + 1);
- target.CopyFrom<value_type>(source_, platform::CPUPlace());
+ target.CopyFrom<value_type>(slice, platform::CPUPlace(),
+ platform::CPUDeviceContext());
}
return result;
@@ -269,7 +279,8 @@ LoDTensor PackDynamicBatch(const std::vector<LoDTensor>& source,
if (index >= seq_meta.end) break;
auto source_ = source[batch_id].Slice<float>(seq_id, seq_id + 1);
auto target = result.Slice<float>(index, index + 1);
- target.CopyFrom<float>(source_, platform::CPUPlace());
+ target.CopyFrom<float>(source_, platform::CPUPlace(),
+ platform::CPUDeviceContext());
}
}
diff --git a/paddle/framework/tensor_array.h b/paddle/framework/tensor_array.h
index 293da04997..046ecb5221 100644
--- a/paddle/framework/tensor_array.h
+++ b/paddle/framework/tensor_array.h
@@ -34,6 +34,13 @@ struct DySeqMeta {
size_t ori_idx;
};
+using DySeqMetaBatch = std::vector<DySeqMeta>;
+
+/*
+ * Extract the indices of instances.
+ */
+std::vector<size_t> GenDyBatchIndice(const DySeqMetaBatch &metas, int batch_id);
+
/*
* TensorArray is a C-array-like array of tensors, it is meant to be used with
* dynamic iteration primitives such as while_loop. It is used to segment inputs
@@ -69,7 +76,7 @@ class TensorArray {
* Recover the original LoD-arranged LoDTensor with the `values`, `level` and
* `indice_map`.
*/
- LoDTensor Pack(size_t level, const std::vector<DySeqMeta> &meta,
+ LoDTensor Pack(size_t level, const DySeqMetaBatch &meta,
const LoD &lod) const;
/*
@@ -77,8 +84,7 @@ class TensorArray {
* `values`, if set `desend`, will sort by length in descending order else in
* ascending order.
*/
- std::vector<DySeqMeta> Unpack(const LoDTensor &source, int level,
- bool length_desend);
+ DySeqMetaBatch Unpack(const LoDTensor &source, int level, bool length_desend);
/*
* Pack the values into a tensor with rank one higher than each tensor in
diff --git a/paddle/framework/tensor_impl.h b/paddle/framework/tensor_impl.h
index 8ee9941982..ce73e0a9ed 100644
--- a/paddle/framework/tensor_impl.h
+++ b/paddle/framework/tensor_impl.h
@@ -88,7 +88,8 @@ inline Tensor& Tensor::ShareDataWith(const Tensor& src) {
template <typename T>
inline void Tensor::CopyFrom(const Tensor& src,
- const platform::Place& dst_place) {
+ const platform::Place& dst_place,
+ const platform::DeviceContext& ctx) {
src.check_memory_size<T>();
Resize(src.dims());
@@ -106,26 +107,45 @@ inline void Tensor::CopyFrom(const Tensor& src,
#ifdef PADDLE_WITH_CUDA
else if (platform::is_gpu_place(src_place) &&
platform::is_cpu_place(dst_place)) {
- memory::Copy(boost::get<platform::CPUPlace>(dst_place), dst_ptr,
- boost::get<platform::GPUPlace>(src_place), src_ptr, size, 0);
+ auto src_gpu_place = boost::get<platform::GPUPlace>(src_place);
+ auto dst_cpu_place = boost::get<platform::CPUPlace>(dst_place);
+ auto ctx_place = ctx.GetPlace();
+ PADDLE_ENFORCE(platform::is_gpu_place(ctx_place));
+ auto ctx_gpu_place = boost::get<platform::GPUPlace>(ctx_place);
+ PADDLE_ENFORCE_EQ(src_gpu_place, ctx_gpu_place);
+ memory::Copy(
+ dst_cpu_place, dst_ptr, src_gpu_place, src_ptr, size,
+ reinterpret_cast<const platform::CUDADeviceContext&>(ctx).stream());
} else if (platform::is_cpu_place(src_place) &&
platform::is_gpu_place(dst_place)) {
- memory::Copy(boost::get<platform::GPUPlace>(dst_place), dst_ptr,
- boost::get<platform::CPUPlace>(src_place), src_ptr, size, 0);
+ auto src_cpu_place = boost::get<platform::CPUPlace>(src_place);
+ auto dst_gpu_place = boost::get<platform::GPUPlace>(dst_place);
+ auto ctx_place = ctx.GetPlace();
+ PADDLE_ENFORCE(platform::is_gpu_place(ctx_place));
+ auto ctx_gpu_place = boost::get<platform::GPUPlace>(ctx_place);
+ PADDLE_ENFORCE_EQ(dst_gpu_place, ctx_gpu_place);
+ memory::Copy(
+ dst_gpu_place, dst_ptr, src_cpu_place, src_ptr, size,
+ reinterpret_cast<const platform::CUDADeviceContext&>(ctx).stream());
} else if (platform::is_gpu_place(src_place) &&
platform::is_gpu_place(dst_place)) {
- memory::Copy(boost::get<platform::GPUPlace>(dst_place), dst_ptr,
- boost::get<platform::GPUPlace>(src_place), src_ptr, size, 0);
+ auto src_gpu_place = boost::get<platform::GPUPlace>(src_place);
+ auto dst_gpu_place = boost::get<platform::GPUPlace>(dst_place);
+ auto ctx_place = ctx.GetPlace();
+ PADDLE_ENFORCE(platform::is_gpu_place(ctx_place));
+ auto ctx_gpu_place = boost::get<platform::GPUPlace>(ctx_place);
+ PADDLE_ENFORCE_EQ(src_gpu_place, ctx_gpu_place);
+ memory::Copy(
+ dst_gpu_place, dst_ptr, src_gpu_place, src_ptr, size,
+ reinterpret_cast<const platform::CUDADeviceContext&>(ctx).stream());
}
- PADDLE_ENFORCE(cudaStreamSynchronize(0),
- "cudaStreamSynchronize failed in Tensor CopyFrom");
-
#endif
}
template <typename T>
inline void Tensor::CopyFromVector(const std::vector<T>& src,
- const platform::Place& dst_place) {
+ const platform::DeviceContext& ctx) {
+ auto dst_place = ctx.GetPlace();
auto src_ptr = static_cast<const void*>(src.data());
platform::CPUPlace src_place;
auto dst_ptr = static_cast<void*>(mutable_data<T>(dst_place));
@@ -137,12 +157,11 @@ inline void Tensor::CopyFromVector(const std::vector<T>& src,
}
#ifdef PADDLE_WITH_CUDA
else if (platform::is_gpu_place(dst_place)) {
- memory::Copy(boost::get<platform::GPUPlace>(dst_place), dst_ptr, src_place,
- src_ptr, size, 0);
+ memory::Copy(
+ boost::get<platform::GPUPlace>(dst_place), dst_ptr, src_place, src_ptr,
+ size,
+ reinterpret_cast<const platform::CUDADeviceContext&>(ctx).stream());
}
- PADDLE_ENFORCE(cudaStreamSynchronize(0),
- "cudaStreamSynchronize failed in Tensor CopyFromVector");
-
#endif
}
diff --git a/paddle/framework/tensor_test.cc b/paddle/framework/tensor_test.cc
index 492eba69e1..0b62fe08ce 100644
--- a/paddle/framework/tensor_test.cc
+++ b/paddle/framework/tensor_test.cc
@@ -194,6 +194,7 @@ TEST(Tensor, CopyFrom) {
{
Tensor src_tensor;
Tensor dst_tensor;
+ CPUDeviceContext cpu_ctx((CPUPlace()));
int* src_ptr = src_tensor.mutable_data<int>(make_ddim({3, 3}), CPUPlace());
@@ -201,7 +202,7 @@ TEST(Tensor, CopyFrom) {
memcpy(src_ptr, arr, 9 * sizeof(int));
auto cpu_place = new paddle::platform::CPUPlace();
- dst_tensor.CopyFrom<int>(src_tensor, *cpu_place);
+ dst_tensor.CopyFrom<int>(src_tensor, *cpu_place, cpu_ctx);
const int* dst_ptr = dst_tensor.data<int>();
ASSERT_NE(src_ptr, dst_ptr);
@@ -210,7 +211,7 @@ TEST(Tensor, CopyFrom) {
}
Tensor slice_tensor = src_tensor.Slice<int>(1, 2);
- dst_tensor.CopyFrom<int>(slice_tensor, *cpu_place);
+ dst_tensor.CopyFrom<int>(slice_tensor, *cpu_place, cpu_ctx);
const int* slice_ptr = slice_tensor.data<int>();
dst_ptr = dst_tensor.data<int>();
ASSERT_NE(dst_ptr, slice_ptr);
@@ -231,13 +232,15 @@ TEST(Tensor, CopyFrom) {
// CPU Tensor to GPU Tensor
auto gpu_place = new paddle::platform::GPUPlace(0);
- gpu_tensor.CopyFrom<int>(src_tensor, *gpu_place);
+ CUDADeviceContext gpu_ctx(*gpu_place);
+ gpu_tensor.CopyFrom<int>(src_tensor, *gpu_place, gpu_ctx);
// GPU Tensor to CPU Tensor
auto cpu_place = new paddle::platform::CPUPlace();
- dst_tensor.CopyFrom<int>(gpu_tensor, *cpu_place);
+ dst_tensor.CopyFrom<int>(gpu_tensor, *cpu_place, gpu_ctx);
- // Compare Tensors
+ // Sync before Compare Tensors
+ gpu_ctx.Wait();
const int* dst_ptr = dst_tensor.data<int>();
ASSERT_NE(src_ptr, dst_ptr);
for (size_t i = 0; i < 9; ++i) {
@@ -247,12 +250,13 @@ TEST(Tensor, CopyFrom) {
Tensor slice_tensor = src_tensor.Slice<int>(1, 2);
// CPU Slice Tensor to GPU Tensor
- gpu_tensor.CopyFrom<int>(slice_tensor, *gpu_place);
+ gpu_tensor.CopyFrom<int>(slice_tensor, *gpu_place, gpu_ctx);
// GPU Tensor to CPU Tensor
- dst_tensor.CopyFrom<int>(gpu_tensor, *cpu_place);
+ dst_tensor.CopyFrom<int>(gpu_tensor, *cpu_place, gpu_ctx);
- // Compare Slice Tensors
+ // Sync before Compare Slice Tensors
+ gpu_ctx.Wait();
const int* slice_ptr = slice_tensor.data<int>();
dst_ptr = dst_tensor.data<int>();
ASSERT_NE(dst_ptr, slice_ptr);
@@ -273,7 +277,8 @@ TEST(Tensor, CopyFromVector) {
// Copy to CPU Tensor
cpu_tensor.Resize(make_ddim({3, 3}));
auto cpu_place = new paddle::platform::CPUPlace();
- cpu_tensor.CopyFromVector<int>(src_vec, *cpu_place);
+ CPUDeviceContext cpu_ctx(*cpu_place);
+ cpu_tensor.CopyFromVector<int>(src_vec, cpu_ctx);
// Compare Tensors
const int* cpu_ptr = cpu_tensor.data<int>();
@@ -285,7 +290,7 @@ TEST(Tensor, CopyFromVector) {
src_vec.erase(src_vec.begin(), src_vec.begin() + 5);
cpu_tensor.Resize(make_ddim({2, 2}));
- cpu_tensor.CopyFromVector<int>(src_vec, *cpu_place);
+ cpu_tensor.CopyFromVector<int>(src_vec, cpu_ctx);
cpu_ptr = cpu_tensor.data<int>();
src_ptr = src_vec.data();
ASSERT_NE(src_ptr, cpu_ptr);
@@ -306,16 +311,19 @@ TEST(Tensor, CopyFromVector) {
// Copy to CPU Tensor
cpu_tensor.Resize(make_ddim({3, 3}));
auto cpu_place = new paddle::platform::CPUPlace();
- cpu_tensor.CopyFromVector<int>(src_vec, *cpu_place);
+ CPUDeviceContext cpu_ctx(*cpu_place);
+ cpu_tensor.CopyFromVector<int>(src_vec, cpu_ctx);
// Copy to GPUTensor
gpu_tensor.Resize(make_ddim({3, 3}));
auto gpu_place = new paddle::platform::GPUPlace();
- gpu_tensor.CopyFromVector<int>(src_vec, *gpu_place);
+ CUDADeviceContext gpu_ctx(*gpu_place);
+ gpu_tensor.CopyFromVector<int>(src_vec, gpu_ctx);
// Copy from GPU to CPU tensor for comparison
- dst_tensor.CopyFrom<int>(gpu_tensor, *cpu_place);
+ dst_tensor.CopyFrom<int>(gpu_tensor, *cpu_place, gpu_ctx);
- // Compare Tensors
+ // Sync before Compare Tensors
+ gpu_ctx.Wait();
const int* src_ptr = src_vec.data();
const int* cpu_ptr = cpu_tensor.data<int>();
const int* dst_ptr = dst_tensor.data<int>();
@@ -329,11 +337,13 @@ TEST(Tensor, CopyFromVector) {
src_vec.erase(src_vec.begin(), src_vec.begin() + 5);
cpu_tensor.Resize(make_ddim({2, 2}));
- cpu_tensor.CopyFromVector<int>(src_vec, *cpu_place);
+ cpu_tensor.CopyFromVector<int>(src_vec, cpu_ctx);
gpu_tensor.Resize(make_ddim({2, 2}));
- gpu_tensor.CopyFromVector<int>(src_vec, *gpu_place);
- dst_tensor.CopyFrom<int>(gpu_tensor, *cpu_place);
+ gpu_tensor.CopyFromVector<int>(src_vec, gpu_ctx);
+ dst_tensor.CopyFrom<int>(gpu_tensor, *cpu_place, gpu_ctx);
+ // Sync before Compare Tensors
+ gpu_ctx.Wait();
src_ptr = src_vec.data();
cpu_ptr = cpu_tensor.data<int>();
dst_ptr = dst_tensor.data<int>();
diff --git a/paddle/framework/type_defs.h b/paddle/framework/type_defs.h
index 6f65a942ba..7e1b79c97b 100644
--- a/paddle/framework/type_defs.h
+++ b/paddle/framework/type_defs.h
@@ -36,8 +36,8 @@ using OpCreator = std::function<OperatorBase*(
const std::string& /*type*/, const VariableNameMap& /*inputs*/,
const VariableNameMap& /*outputs*/, const AttributeMap& /*attrs*/)>;
-using GradOpMakerFN =
- std::function<std::vector<std::unique_ptr<OpDescBind>>(const OpDescBind&)>;
+using GradOpMakerFN = std::function<std::vector<std::unique_ptr<OpDescBind>>(
+ const OpDescBind&, const std::unordered_set<std::string>& /*no_grad_set*/)>;
} // namespace framework
} // namespace paddle
diff --git a/paddle/framework/var_desc.h b/paddle/framework/var_desc.h
index 464fece85f..4436879564 100644
--- a/paddle/framework/var_desc.h
+++ b/paddle/framework/var_desc.h
@@ -34,6 +34,7 @@ inline std::vector<T> RepeatedToVector(
template <typename T, typename RepeatedField>
inline void VectorToRepeated(const std::vector<T> &vec,
RepeatedField *repeated_field) {
+ repeated_field->Clear();
repeated_field->Reserve(vec.size());
for (const auto &elem : vec) {
*repeated_field->Add() = elem;
@@ -44,6 +45,7 @@ inline void VectorToRepeated(const std::vector<T> &vec,
template <typename RepeatedField>
inline void VectorToRepeated(const std::vector<bool> &vec,
RepeatedField *repeated_field) {
+ repeated_field->Clear();
repeated_field->Reserve(vec.size());
for (auto elem : vec) {
*repeated_field->Add() = elem;
diff --git a/paddle/operators/CMakeLists.txt b/paddle/operators/CMakeLists.txt
index 7dae8fe2f9..ad941bde2b 100644
--- a/paddle/operators/CMakeLists.txt
+++ b/paddle/operators/CMakeLists.txt
@@ -112,7 +112,9 @@ set(DEPS_OPS
cond_op
cross_entropy_op
softmax_with_cross_entropy_op
- sum_op)
+ sum_op
+ pool_op
+ pool_with_index_op)
op_library(recurrent_op SRCS recurrent_op.cc rnn/recurrent_op_utils.cc
@@ -121,6 +123,8 @@ op_library(cond_op SRCS cond_op.cc DEPS framework_proto tensor operator net_op)
op_library(cross_entropy_op DEPS cross_entropy)
op_library(softmax_with_cross_entropy_op DEPS cross_entropy softmax)
op_library(sum_op DEPS net_op)
+op_library(pool_op DEPS pooling)
+op_library(pool_with_index_op DEPS pooling)
list(REMOVE_ITEM GENERAL_OPS ${DEPS_OPS})
foreach(src ${GENERAL_OPS})
diff --git a/paddle/operators/activation_op.cc b/paddle/operators/activation_op.cc
index a6bb738af3..84c3775b4f 100644
--- a/paddle/operators/activation_op.cc
+++ b/paddle/operators/activation_op.cc
@@ -137,6 +137,24 @@ class TanhShrinkOpMaker : public framework::OpProtoAndCheckerMaker {
}
};
+template <typename AttrType>
+class HardShrinkOpMaker : public framework::OpProtoAndCheckerMaker {
+ public:
+ HardShrinkOpMaker(framework::OpProto *proto,
+ framework::OpAttrChecker *op_checker)
+ : OpProtoAndCheckerMaker(proto, op_checker) {
+ AddInput("X", "Input of HardShrink operator");
+ AddOutput("Y", "Output of HardShrink operator");
+ AddComment(
+ "HardShrink activation operator, "
+ "hard_shrink(x) = x if x > lambda"
+ "hard_shrink(x) = x if x < -lambda"
+ "hard_shrink(x) = 0 otherwise");
+ AddAttr<AttrType>("threshold", "The value of threshold for HardShrink")
+ .SetDefault(static_cast<AttrType>(0.5));
+ }
+};
+
class SqrtOpMaker : public framework::OpProtoAndCheckerMaker {
public:
SqrtOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker)
@@ -188,6 +206,17 @@ class SquareOpMaker : public framework::OpProtoAndCheckerMaker {
}
};
+class SoftplusOpMaker : public framework::OpProtoAndCheckerMaker {
+ public:
+ SoftplusOpMaker(framework::OpProto *proto,
+ framework::OpAttrChecker *op_checker)
+ : OpProtoAndCheckerMaker(proto, op_checker) {
+ AddInput("X", "Input of Softplus operator");
+ AddOutput("Y", "Output of Softplus operator");
+ AddComment("Softplus activation operator, softplus(x) = log(1 + exp(x))");
+ }
+};
+
class SoftsignOpMaker : public framework::OpProtoAndCheckerMaker {
public:
SoftsignOpMaker(framework::OpProto *proto,
@@ -292,6 +321,55 @@ class STanhOpMaker : public framework::OpProtoAndCheckerMaker {
}
};
+template <typename AttrType>
+class ThresholdedReluOpMaker : public framework::OpProtoAndCheckerMaker {
+ public:
+ ThresholdedReluOpMaker(framework::OpProto *proto,
+ framework::OpAttrChecker *op_checker)
+ : OpProtoAndCheckerMaker(proto, op_checker) {
+ AddInput("X", "Input of ThresholdedRelu operator");
+ AddOutput("Y", "Output of ThresholdedRelu operator");
+ AddComment(
+ "ThresholdedRelu activation operator, "
+ "thresholded_relu = x for x > threshold, "
+ "thresholded_relu = 0 otherwise.");
+ AddAttr<AttrType>("threshold", "The threshold location of activation")
+ .SetDefault(static_cast<AttrType>(1.0));
+ }
+};
+
+template <typename AttrType>
+class HardSigmoidOpMaker : public framework::OpProtoAndCheckerMaker {
+ public:
+ HardSigmoidOpMaker(framework::OpProto *proto,
+ framework::OpAttrChecker *op_checker)
+ : OpProtoAndCheckerMaker(proto, op_checker) {
+ AddInput("X", "Input of HardSigmoid operator");
+ AddOutput("Y", "Output of HardSigmoid operator");
+ AddComment(R"DOC(
+Hard Sigmoid activation operator.
+
+Segment-wise linear approximation of sigmoid[1].
+This is much faster than sigmoid.
+
+hard_sigmoid = max(0, min(1, slope * x + shift))
+
+The slope should be positive. The offset can be either positive or negative.
+The default slope and shift are set from [1].
+It is recommended to use the defaults for this activation.
+
+References:
+ [1] Noisy Activation Functions
+ (https://arxiv.org/abs/1603.00391)
+
+ )DOC");
+ AddAttr<AttrType>("slope", "Slope for linear approximation of sigmoid")
+ .SetDefault(static_cast<AttrType>(0.2));
+ AddAttr<AttrType>("offset", "Offset for linear approximation of sigmoid")
+ .SetDefault(static_cast<AttrType>(0.5));
+ }
+};
+
} // namespace operators
} // namespace paddle
@@ -333,6 +411,9 @@ REGISTER_OP(log, ops::ActivationOp, ops::LogOpMaker, log_grad,
REGISTER_OP(square, ops::ActivationOp, ops::SquareOpMaker, square_grad,
ops::ActivationOpGrad);
+REGISTER_OP(softplus, ops::ActivationOp, ops::SoftplusOpMaker, softplus_grad,
+ ops::ActivationOpGrad);
+
REGISTER_OP(softsign, ops::ActivationOp, ops::SoftsignOpMaker, softsign_grad,
ops::ActivationOpGrad);
@@ -357,6 +438,16 @@ REGISTER_OP(pow, ops::ActivationOp, ops::PowOpMaker<float>, pow_grad,
REGISTER_OP(stanh, ops::ActivationOp, ops::STanhOpMaker<float>, stanh_grad,
ops::ActivationOpGrad);
+REGISTER_OP(hard_shrink, ops::ActivationOp, ops::HardShrinkOpMaker<float>,
+ hard_shrink_grad, ops::ActivationOpGrad);
+
+REGISTER_OP(thresholded_relu, ops::ActivationOp,
+ ops::ThresholdedReluOpMaker<float>, thresholded_relu_grad,
+ ops::ActivationOpGrad);
+
+REGISTER_OP(hard_sigmoid, ops::ActivationOp, ops::HardSigmoidOpMaker<float>,
+ hard_sigmoid_grad, ops::ActivationOpGrad);
+
#define REGISTER_ACTIVATION_CPU_KERNEL(act_type, functor, grad_functor) \
REGISTER_OP_CPU_KERNEL( \
act_type, \
diff --git a/paddle/operators/activation_op.h b/paddle/operators/activation_op.h
index 70d5a62052..4f4eb44fed 100644
--- a/paddle/operators/activation_op.h
+++ b/paddle/operators/activation_op.h
@@ -199,6 +199,39 @@ struct TanhShrinkGradFunctor : public BaseActivationFunctor<T> {
}
};
+// tanhshrink(x) = x - tanh(x)
+// where tanh(x) = (exp(x) - exp(-x)) / (exp(x) + exp(-x))
+template <typename T>
+struct HardShrinkFunctor : public BaseActivationFunctor<T> {
+ float threshold;
+
+ typename BaseActivationFunctor<T>::AttrPair GetAttrs() {
+ return {{"threshold", &threshold}};
+ }
+ template <typename Device, typename X, typename Y>
+ void operator()(Device d, X x, Y y) const {
+ auto temp1 = (x < (threshold * -1)).template cast<T>().eval();
+ auto temp2 = (x > threshold).template cast<T>().eval();
+ y.device(d) = x * (temp1 + temp2);
+ }
+};
+
+template <typename T>
+struct HardShrinkGradFunctor : public BaseActivationFunctor<T> {
+ float threshold;
+
+ typename BaseActivationFunctor<T>::AttrPair GetAttrs() {
+ return {{"threshold", &threshold}};
+ }
+
+ template <typename Device, typename X, typename Y, typename dY, typename dX>
+ void operator()(Device d, X x, Y y, dY dy, dX dx) const {
+ auto temp1 = (x < (threshold * -1)).template cast<T>().eval();
+ auto temp2 = (x > threshold).template cast<T>().eval();
+ dx.device(d) = dy * (temp1 + temp2).template cast<T>();
+ }
+};
+
// softshrink(x) = x - lambda, if x > lambda; x + lambda, if x < lambda; 0
// otherwise
template <typename T>
@@ -351,8 +384,6 @@ template <typename T>
struct Relu6Functor : public BaseActivationFunctor<T> {
float threshold;
- // NOTE: Explicit hides the `BaseActivationFunctor<T>::GetAttrs`
- // not polymorphism for speed.
typename BaseActivationFunctor<T>::AttrPair GetAttrs() {
return {{"threshold", &threshold}};
}
@@ -376,6 +407,33 @@ struct Relu6GradFunctor : public BaseActivationFunctor<T> {
}
};
+// softplus(x) = log(1 + exp(x))
+// When x is a very large positive number, exp(x) may explode to inf,
+// Using trick below for numerical stability
+// https://hips.seas.harvard.edu/blog/2013/01/09/computing-log-sum-exp/
+// Then: softplus(x) = max(x, 0) + log(exp(-max(x, 0)) + exp(x - max(x, 0)))
+template <typename T>
+struct SoftplusFunctor : public BaseActivationFunctor<T> {
+ template <typename Device, typename X, typename Y>
+ void operator()(Device d, X x, Y y) {
+ auto temp = x.cwiseMax(static_cast<T>(0)); // temp = max(x, 0)
+ y.device(d) = temp + (((-temp).exp() + (x - temp).exp()).log());
+ }
+};
+
+// d(softplus(x))/dx = exp(x) / (1 + exp(x))
+// For numerical stability:
+// d(softplus(x))/dx = exp(x - max(x, 0)) / (exp(-max(x, 0)) +
+// exp(x - max(x, 0)))
+template <typename T>
+struct SoftplusGradFunctor : public BaseActivationFunctor<T> {
+ template <typename Device, typename X, typename Y, typename dY, typename dX>
+ void operator()(Device d, X x, Y y, dY dy, dX dx) {
+ auto temp = x.cwiseMax(static_cast<T>(0)); // temp = max(x, 0)
+ dx.device(d) = dy * ((x - temp).exp() / ((-temp).exp() + (x - temp).exp()));
+ }
+};
+
// softsign(x) = x / (1 + |x|)
template <typename T>
struct SoftsignFunctor : public BaseActivationFunctor<T> {
@@ -532,27 +590,89 @@ struct STanhGradFunctor : public BaseActivationFunctor<T> {
}
};
+template <typename T>
+struct ThresholdedReluFunctor : public BaseActivationFunctor<T> {
+ float threshold;
+ typename BaseActivationFunctor<T>::AttrPair GetAttrs() {
+ return {{"threshold", &threshold}};
+ }
+
+ template <typename Device, typename X, typename Y>
+ void operator()(Device d, X x, Y y) const {
+ y.device(d) = (x > static_cast<T>(threshold)).template cast<T>() * x;
+ }
+};
+
+template <typename T>
+struct ThresholdedReluGradFunctor : public BaseActivationFunctor<T> {
+ float threshold;
+ typename BaseActivationFunctor<T>::AttrPair GetAttrs() {
+ return {{"threshold", &threshold}};
+ }
+
+ template <typename Device, typename X, typename Y, typename dY, typename dX>
+ void operator()(Device d, X x, Y y, dY dy, dX dx) const {
+ dx.device(d) = dy * (x > static_cast<T>(threshold)).template cast<T>();
+ }
+};
+
+template <typename T>
+struct HardSigmoidFunctor : public BaseActivationFunctor<T> {
+ float slope;
+ float offset;
+ typename BaseActivationFunctor<T>::AttrPair GetAttrs() {
+ return {{"slope", &slope}, {"offset", &offset}};
+ }
+
+ template <typename Device, typename X, typename Y>
+ void operator()(Device d, X x, Y y) const {
+ auto temp = x * static_cast<T>(slope) + static_cast<T>(offset);
+ y.device(d) = temp.cwiseMax(static_cast<T>(0)).cwiseMin(static_cast<T>(1));
+ }
+};
+
+template <typename T>
+struct HardSigmoidGradFunctor : public BaseActivationFunctor<T> {
+ float slope;
+ float offset;
+ typename BaseActivationFunctor<T>::AttrPair GetAttrs() {
+ return {{"slope", &slope}, {"offset", &offset}};
+ }
+
+ template <typename Device, typename X, typename Y, typename dY, typename dX>
+ void operator()(Device d, X x, Y y, dY dy, dX dx) const {
+ dx.device(d) =
+ dy *
+ ((y > static_cast<T>(0)) * (y < static_cast<T>(1))).template cast<T>() *
+ static_cast<T>(slope);
+ }
+};
+
} // namespace operators
} // namespace paddle
-#define FOR_EACH_KERNEL_FUNCTOR(__macro) \
- __macro(sigmoid, SigmoidFunctor, SigmoidGradFunctor); \
- __macro(logsigmoid, LogSigmoidFunctor, LogSigmoidGradFunctor); \
- __macro(exp, ExpFunctor, ExpGradFunctor); \
- __macro(relu, ReluFunctor, ReluGradFunctor); \
- __macro(tanh, TanhFunctor, TanhGradFunctor); \
- __macro(softshrink, SoftShrinkFunctor, SoftShrinkGradFunctor); \
- __macro(sqrt, SqrtFunctor, SqrtGradFunctor); \
- __macro(abs, AbsFunctor, AbsGradFunctor); \
- __macro(reciprocal, ReciprocalFunctor, ReciprocalGradFunctor); \
- __macro(log, LogFunctor, LogGradFunctor); \
- __macro(square, SquareFunctor, SquareGradFunctor); \
- __macro(brelu, BReluFunctor, BReluGradFunctor); \
- __macro(soft_relu, SoftReluFunctor, SoftReluGradFunctor); \
- __macro(pow, PowFunctor, PowGradFunctor); \
- __macro(stanh, STanhFunctor, STanhGradFunctor); \
- __macro(softsign, SoftsignFunctor, SoftsignGradFunctor); \
- __macro(relu6, Relu6Functor, Relu6GradFunctor); \
- __macro(leaky_relu, LeakyReluFunctor, LeakyReluGradFunctor); \
- __macro(tanh_shrink, TanhShrinkFunctor, TanhShrinkGradFunctor); \
- __macro(elu, ELUFunctor, ELUGradFunctor)
+#define FOR_EACH_KERNEL_FUNCTOR(__macro) \
+ __macro(sigmoid, SigmoidFunctor, SigmoidGradFunctor); \
+ __macro(logsigmoid, LogSigmoidFunctor, LogSigmoidGradFunctor); \
+ __macro(exp, ExpFunctor, ExpGradFunctor); \
+ __macro(relu, ReluFunctor, ReluGradFunctor); \
+ __macro(tanh, TanhFunctor, TanhGradFunctor); \
+ __macro(softshrink, SoftShrinkFunctor, SoftShrinkGradFunctor); \
+ __macro(sqrt, SqrtFunctor, SqrtGradFunctor); \
+ __macro(abs, AbsFunctor, AbsGradFunctor); \
+ __macro(reciprocal, ReciprocalFunctor, ReciprocalGradFunctor); \
+ __macro(log, LogFunctor, LogGradFunctor); \
+ __macro(square, SquareFunctor, SquareGradFunctor); \
+ __macro(brelu, BReluFunctor, BReluGradFunctor); \
+ __macro(soft_relu, SoftReluFunctor, SoftReluGradFunctor); \
+ __macro(pow, PowFunctor, PowGradFunctor); \
+ __macro(stanh, STanhFunctor, STanhGradFunctor); \
+ __macro(softplus, SoftplusFunctor, SoftplusGradFunctor); \
+ __macro(softsign, SoftsignFunctor, SoftsignGradFunctor); \
+ __macro(relu6, Relu6Functor, Relu6GradFunctor); \
+ __macro(leaky_relu, LeakyReluFunctor, LeakyReluGradFunctor); \
+ __macro(tanh_shrink, TanhShrinkFunctor, TanhShrinkGradFunctor); \
+ __macro(elu, ELUFunctor, ELUGradFunctor); \
+ __macro(hard_shrink, HardShrinkFunctor, HardShrinkGradFunctor); \
+ __macro(hard_sigmoid, HardSigmoidFunctor, HardSigmoidGradFunctor); \
+ __macro(thresholded_relu, ThresholdedReluFunctor, ThresholdedReluGradFunctor);
diff --git a/paddle/operators/adam_op.cc b/paddle/operators/adam_op.cc
new file mode 100644
index 0000000000..293b37b775
--- /dev/null
+++ b/paddle/operators/adam_op.cc
@@ -0,0 +1,144 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/operators/adam_op.h"
+
+namespace paddle {
+namespace operators {
+
+class AdamOp : public framework::OperatorWithKernel {
+ public:
+ using framework::OperatorWithKernel::OperatorWithKernel;
+
+ protected:
+ void InferShape(framework::InferShapeContext *ctx) const override {
+ PADDLE_ENFORCE(ctx->HasInput("Param"),
+ "Input(Param) of AdamOp should not be null.");
+ PADDLE_ENFORCE(ctx->HasInput("Grad"),
+ "Input(Grad) of AdamOp should not be null.");
+ PADDLE_ENFORCE(ctx->HasInput("Moment1"),
+ "Input(Moment1) of AdamOp should not be null.");
+ PADDLE_ENFORCE(ctx->HasInput("Moment2"),
+ "Input(Moment2) of AdamOp should not be null.");
+ PADDLE_ENFORCE(ctx->HasInput("LearningRate"),
+ "Input(LearningRate) of AdamOp should not be null.");
+ PADDLE_ENFORCE(ctx->HasInput("Beta1Pow"),
+ "Input(Beta1Pow) of AdamOp should not be null.");
+ PADDLE_ENFORCE(ctx->HasInput("Beta2Pow"),
+ "Input(Beta2Pow) of AdamOp should not be null.");
+
+ PADDLE_ENFORCE(ctx->HasOutput("ParamOut"),
+ "Output(ParamOut) of AdamOp should not be null.");
+ PADDLE_ENFORCE(ctx->HasOutput("Moment1Out"),
+ "Output(Moment1Out) of AdamOp should not be null.");
+ PADDLE_ENFORCE(ctx->HasOutput("Moment2Out"),
+ "Output(Moment2Out) of AdamOp should not be null.");
+ PADDLE_ENFORCE(ctx->HasOutput("Beta1PowOut"),
+ "Output(Beta1PowOut) of AdamOp should not be null.");
+ PADDLE_ENFORCE(ctx->HasOutput("Beta2PowOut"),
+ "Output(Beta2PowOut) of AdamOp should not be null.");
+
+ auto lr_dims = ctx->GetInputDim("LearningRate");
+ PADDLE_ENFORCE_EQ(framework::product(lr_dims), 1,
+ "Learning rate should have 1 dimension");
+ auto beta1_pow_dims = ctx->GetInputDim("Beta1Pow");
+ PADDLE_ENFORCE_EQ(framework::product(beta1_pow_dims), 1,
+ "Beta1 power accumulator should have 1 dimension");
+ auto beta2_pow_dims = ctx->GetInputDim("Beta2Pow");
+ PADDLE_ENFORCE_EQ(framework::product(beta1_pow_dims), 1,
+ "Beta1 power accumulator should have 1 dimension");
+
+ auto param_dims = ctx->GetInputDim("Param");
+ PADDLE_ENFORCE_EQ(
+ param_dims, ctx->GetInputDim("Grad"),
+ "Param and Grad input of AdamOp should have same dimension");
+ PADDLE_ENFORCE_EQ(
+ param_dims, ctx->GetInputDim("Moment1"),
+ "Param and Moment input of AdamOp should have same dimension");
+ PADDLE_ENFORCE_EQ(
+ param_dims, ctx->GetInputDim("Moment2"),
+ "Param and InfNorm input of AdamOp should have same dimension");
+
+ ctx->SetOutputDim("ParamOut", param_dims);
+ ctx->SetOutputDim("Moment1Out", param_dims);
+ ctx->SetOutputDim("Moment2Out", param_dims);
+ ctx->SetOutputDim("Beta1PowOut", beta1_pow_dims);
+ ctx->SetOutputDim("Beta2PowOut", beta2_pow_dims);
+ }
+};
+
+class AdamOpMaker : public framework::OpProtoAndCheckerMaker {
+ public:
+ AdamOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker)
+ : OpProtoAndCheckerMaker(proto, op_checker) {
+ AddInput("Param", "(Tensor) Input parameter");
+ AddInput("Grad", "(Tensor) Input gradient");
+ AddInput("LearningRate", "(Tensor) Learning rate");
+ AddInput("Moment1", "(Tensor) Input first moment");
+ AddInput("Moment2", "(Tensor) Input second moment");
+ AddInput("Beta1Pow", "(Tensor) Input beta1 power accumulator");
+ AddInput("Beta2Pow", "(Tensor) Input beta2 power accumulator");
+
+ AddOutput("ParamOut", "(Tensor) Output parameter");
+ AddOutput("Moment1Out", "(Tensor) Output first moment");
+ AddOutput("Moment2Out", "(Tensor) Output second moment");
+ AddOutput("Beta1PowOut", "(Tensor) Output beta1 power accumulator");
+ AddOutput("Beta2PowOut", "(Tensor) Output beta2 power accumulator");
+
+ AddAttr<float>("beta1",
+ "(float, default 0.9) "
+ "Exponential decay rate for the "
+ "first moment estimates.")
+ .SetDefault(0.9f);
+ AddAttr<float>("beta2",
+ "(float, default 0.999) "
+ "exponential decay rate for the "
+ "second moment estimates.")
+ .SetDefault(0.999f);
+ AddAttr<float>("epsilon",
+ "(float, default 1.0e-8) "
+ "Constant for numerical stability")
+ .SetDefault(1.0e-8f);
+
+ AddComment(R"DOC(
+Adam Updates Operator.
+
+This implements the Adam optimizer from Section 2 of the Adam
+paper[1]. Adam is a first-order gradient-based optimization
+method based on adaptive estimates of lower-order moments.
+
+Adam updates:
+
+moment1_out = beta1 * moment1 + (1 − beta1) * grad
+moment2_out = beta2 * moment2 + (1 − beta2) * grad * grad
+beta1_pow_out = beta1_pow * beta1
+beta2_pow_out = beta2_pow * beta2
+learning_rate_t = learning_rate_t *
+ sqrt(1 - beta2_pow_out) / (1 - beta1_pow_out)
+param_out = param - learning_rate_t * moment1/ (sqrt(moment2) + epsilon)
+
+References:
+ [1] Adam: A Method for Stochastic Optimization
+ (https://arxiv.org/abs/1412.6980)
+
+)DOC");
+ }
+};
+} // namespace operators
+} // namespace paddle
+
+namespace ops = paddle::operators;
+REGISTER_OP_WITHOUT_GRADIENT(adam, ops::AdamOp, ops::AdamOpMaker);
+REGISTER_OP_CPU_KERNEL(adam,
+ ops::AdamOpKernel<paddle::platform::CPUPlace, float>);
diff --git a/paddle/operators/adam_op.cu b/paddle/operators/adam_op.cu
new file mode 100644
index 0000000000..a3def912e5
--- /dev/null
+++ b/paddle/operators/adam_op.cu
@@ -0,0 +1,20 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. */
+
+#define EIGEN_USE_GPU
+#include "paddle/operators/adam_op.h"
+
+namespace ops = paddle::operators;
+REGISTER_OP_GPU_KERNEL(adam,
+ ops::AdamOpKernel<paddle::platform::GPUPlace, float>);
diff --git a/paddle/operators/adam_op.h b/paddle/operators/adam_op.h
new file mode 100644
index 0000000000..789c2f14b3
--- /dev/null
+++ b/paddle/operators/adam_op.h
@@ -0,0 +1,82 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+#include "paddle/framework/eigen.h"
+#include "paddle/framework/op_registry.h"
+
+namespace paddle {
+namespace operators {
+
+template <typename Place, typename T>
+class AdamOpKernel : public framework::OpKernel<T> {
+ public:
+ void Compute(const framework::ExecutionContext& ctx) const override {
+ auto param_out_tensor = ctx.Output<framework::Tensor>("ParamOut");
+ auto moment1_out_tensor = ctx.Output<framework::Tensor>("Moment1Out");
+ auto moment2_out_tensor = ctx.Output<framework::Tensor>("Moment2Out");
+ auto beta1_pow_out_tensor = ctx.Output<framework::Tensor>("Beta1PowOut");
+ auto beta2_pow_out_tensor = ctx.Output<framework::Tensor>("Beta2PowOut");
+
+ param_out_tensor->mutable_data<T>(ctx.GetPlace());
+ moment1_out_tensor->mutable_data<T>(ctx.GetPlace());
+ moment2_out_tensor->mutable_data<T>(ctx.GetPlace());
+ beta1_pow_out_tensor->mutable_data<T>(ctx.GetPlace());
+ beta2_pow_out_tensor->mutable_data<T>(ctx.GetPlace());
+
+ float beta1 = ctx.Attr<float>("beta1");
+ float beta2 = ctx.Attr<float>("beta2");
+ float epsilon = ctx.Attr<float>("epsilon");
+
+ auto param = framework::EigenVector<T>::Flatten(
+ *ctx.Input<framework::Tensor>("Param"));
+ auto grad = framework::EigenVector<T>::Flatten(
+ *ctx.Input<framework::Tensor>("Grad"));
+ auto moment1 = framework::EigenVector<T>::Flatten(
+ *ctx.Input<framework::Tensor>("Moment1"));
+ auto moment2 = framework::EigenVector<T>::Flatten(
+ *ctx.Input<framework::Tensor>("Moment2"));
+ auto lr = framework::EigenVector<T>::Flatten(
+ *ctx.Input<framework::Tensor>("LearningRate"));
+ auto beta1_pow = framework::EigenVector<T>::Flatten(
+ *ctx.Input<framework::Tensor>("Beta1Pow"));
+ auto beta2_pow = framework::EigenVector<T>::Flatten(
+ *ctx.Input<framework::Tensor>("Beta2Pow"));
+ auto param_out = framework::EigenVector<T>::Flatten(*param_out_tensor);
+ auto moment1_out = framework::EigenVector<T>::Flatten(*moment1_out_tensor);
+ auto moment2_out = framework::EigenVector<T>::Flatten(*moment2_out_tensor);
+ auto beta1_pow_out =
+ framework::EigenVector<T>::Flatten(*beta1_pow_out_tensor);
+ auto beta2_pow_out =
+ framework::EigenVector<T>::Flatten(*beta2_pow_out_tensor);
+ auto place = ctx.GetEigenDevice<Place>();
+
+ moment1_out.device(place) = beta1 * moment1 + (1 - beta1) * grad;
+ moment2_out.device(place) = beta2 * moment2 + (1 - beta2) * grad.square();
+ beta1_pow_out.device(place) = beta1_pow * beta1;
+ beta2_pow_out.device(place) = beta2_pow * beta2;
+ // All of these are tensors of 1 element
+ auto lr_t = lr * (1 - beta2_pow_out).sqrt() / (1 - beta1_pow_out);
+ // Eigen does not support automatic broadcast
+ // Get dimensions of moment vector to broadcast lr_t
+ Eigen::DSizes<int, 1> m_dsize(moment1_out_tensor->numel());
+ param_out.device(place) =
+ param -
+ lr_t.broadcast(m_dsize) *
+ (moment1_out / (moment2_out.sqrt() + epsilon));
+ }
+};
+
+} // namespace operators
+} // namespace paddle
diff --git a/paddle/operators/conv2d_op.cc b/paddle/operators/conv2d_op.cc
index 6325d4248f..1acb8415d0 100644
--- a/paddle/operators/conv2d_op.cc
+++ b/paddle/operators/conv2d_op.cc
@@ -12,111 +12,91 @@
See the License for the specific language governing permissions and
limitations under the License. */
-#include "paddle/operators/gemm_conv2d_op.h"
+#include "paddle/operators/conv2d_op.h"
namespace paddle {
namespace operators {
-int outputSize(int input_size, int filter_size, int padding, int stride) {
- int output_size = (input_size - filter_size + 2 * padding) / stride + 1;
- return output_size;
+void Conv2DOp::InferShape(framework::InferShapeContext* ctx) const {
+ PADDLE_ENFORCE(ctx->HasInput("Input"),
+ "Input(Input) of Conv2DOp should not be null.");
+ PADDLE_ENFORCE(ctx->HasInput("Filter"),
+ "Input(Filter) of Conv2DOp should not be null.");
+ PADDLE_ENFORCE(ctx->HasOutput("Output"),
+ "Output(Output) of Conv2DOp should not be null.");
+
+ auto in_dims = ctx->GetInputDim("Input");
+ auto filter_dims = ctx->GetInputDim("Filter");
+ std::vector<int> strides = ctx->Attrs().Get<std::vector<int>>("strides");
+ std::vector<int> paddings = ctx->Attrs().Get<std::vector<int>>("paddings");
+ int groups = ctx->Attrs().Get<int>("groups");
+ int input_channels = in_dims[1];
+ int output_channels = filter_dims[0];
+
+ PADDLE_ENFORCE_EQ(in_dims.size(), 4, "Conv2DOp input should be 4-D.");
+ PADDLE_ENFORCE_EQ(filter_dims.size(), 4, "Conv2DOp filter should be 4-D.");
+ PADDLE_ENFORCE_EQ(input_channels, filter_dims[1] * groups,
+ "The number of input channels should be equal to filter "
+ "channels * groups.");
+ PADDLE_ENFORCE_EQ(
+ output_channels % groups, 0,
+ "The number of output channels should be divided by groups.");
+
+ auto output_height =
+ OutputSize(in_dims[2], filter_dims[2], paddings[0], strides[0]);
+ auto output_width =
+ OutputSize(in_dims[3], filter_dims[3], paddings[1], strides[1]);
+ ctx->SetOutputDim("Output",
+ {in_dims[0], filter_dims[0], output_height, output_width});
}
-class Conv2DOp : public framework::OperatorWithKernel {
- public:
- using framework::OperatorWithKernel::OperatorWithKernel;
-
- protected:
- void InferShape(framework::InferShapeContext* ctx) const override {
- PADDLE_ENFORCE(ctx->HasInput("Input"),
- "Input(Input) of Conv2DOp should not be null.");
- PADDLE_ENFORCE(ctx->HasInput("Filter"),
- "Input(Filter) of Conv2DOp should not be null.");
- PADDLE_ENFORCE(ctx->HasOutput("Output"),
- "Output(Output) of Conv2DOp should not be null.");
-
- auto in_dims = ctx->GetInputDim("Input");
- auto filter_dims = ctx->GetInputDim("Filter");
- std::vector<int> strides = ctx->Attrs().Get<std::vector<int>>("strides");
- std::vector<int> paddings = ctx->Attrs().Get<std::vector<int>>("paddings");
- int groups = ctx->Attrs().Get<int>("groups");
- int input_channels = in_dims[1];
- int output_channels = filter_dims[0];
-
- PADDLE_ENFORCE_EQ(in_dims.size(), 4, "Conv2DOp input should be 4-D.");
- PADDLE_ENFORCE_EQ(filter_dims.size(), 4, "Conv2DOp filter should be 4-D.");
- PADDLE_ENFORCE_EQ(input_channels, filter_dims[1] * groups,
- "The number of input channels should be equal to filter "
- "channels * groups.");
- PADDLE_ENFORCE_EQ(
- output_channels % groups, 0,
- "The number of output channels should be divided by groups.");
-
- auto output_height =
- outputSize(in_dims[2], filter_dims[2], paddings[0], strides[0]);
- auto output_width =
- outputSize(in_dims[3], filter_dims[3], paddings[1], strides[1]);
- ctx->SetOutputDim(
- "Output", {in_dims[0], filter_dims[0], output_height, output_width});
- }
-};
-
-class Conv2DOpMaker : public framework::OpProtoAndCheckerMaker {
- public:
- Conv2DOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker)
- : OpProtoAndCheckerMaker(proto, op_checker) {
- AddInput(
- "Input",
- "The input tensor of convolution operator. "
- "The format of input tensor is NCHW. Where N is batch size, C is the "
- "number of channels, H and W is the height and width of image.");
- AddInput(
- "Filter",
- "The filter tensor of convolution operator."
- "The format of the filter tensor is MCHW, where M is the number of "
- "output image channels, C is the number of input image channels, "
- "H and W is height and width of filter. "
- "If the groups attribute is greater than 1, C equal the number of "
- "input image channels divided by the groups.");
- AddOutput("Output",
- "The output tensor of convolution operator."
- "The format of output tensor is also NCHW.");
- AddAttr<std::vector<int>>("strides", "strides of convolution operator.")
- .SetDefault({1, 1});
- AddAttr<std::vector<int>>("paddings", "paddings of convolution operator.")
- .SetDefault({0, 0});
- AddAttr<int>(
- "groups",
- "group size of convolution operator. "
- "Refer to grouped convolution in Alex Krizhevsky's paper: "
- "when group=2, the first half of the filters are only connected to the "
- "first half of the input channels, and the second half only connected "
- "to the second half.")
- .SetDefault(1);
- AddComment(R"DOC(
+Conv2DOpMaker::Conv2DOpMaker(framework::OpProto* proto,
+ framework::OpAttrChecker* op_checker)
+ : OpProtoAndCheckerMaker(proto, op_checker) {
+ AddInput(
+ "Input",
+ "The input tensor of convolution operator. "
+ "The format of input tensor is NCHW. Where N is batch size, C is the "
+ "number of channels, H and W is the height and width of image.");
+ AddInput("Filter",
+ "The filter tensor of convolution operator."
+ "The format of the filter tensor is MCHW, where M is the number of "
+ "output image channels, C is the number of input image channels, "
+ "H and W is height and width of filter. "
+ "If the groups attribute is greater than 1, C equal the number of "
+ "input image channels divided by the groups.");
+ AddOutput("Output",
+ "The output tensor of convolution operator."
+ "The format of output tensor is also NCHW.");
+ AddAttr<std::vector<int>>("strides", "strides of convolution operator.")
+ .SetDefault({1, 1});
+ AddAttr<std::vector<int>>("paddings", "paddings of convolution operator.")
+ .SetDefault({0, 0});
+ AddAttr<int>(
+ "groups",
+ "group size of convolution operator. "
+ "Refer to grouped convolution in Alex Krizhevsky's paper: "
+ "when group=2, the first half of the filters are only connected to the "
+ "first half of the input channels, and the second half only connected "
+ "to the second half.")
+ .SetDefault(1);
+ AddComment(R"DOC(
The convolution operation calculates the output based on the input, filter
and strides, paddings, groups parameters. The size of each dimension of the
parameters is checked in the infer-shape.
)DOC");
- }
-};
-
-class Conv2DOpGrad : public framework::OperatorWithKernel {
- public:
- using framework::OperatorWithKernel::OperatorWithKernel;
+}
- protected:
- void InferShape(framework::InferShapeContext* ctx) const override {
- auto in_dims = ctx->GetInputDim("Input");
- auto filter_dims = ctx->GetInputDim("Filter");
- if (ctx->HasOutput(framework::GradVarName("Input"))) {
- ctx->SetOutputDim(framework::GradVarName("Input"), in_dims);
- }
- if (ctx->HasOutput(framework::GradVarName("Filter"))) {
- ctx->SetOutputDim(framework::GradVarName("Filter"), filter_dims);
- }
+void Conv2DOpGrad::InferShape(framework::InferShapeContext* ctx) const {
+ auto in_dims = ctx->GetInputDim("Input");
+ auto filter_dims = ctx->GetInputDim("Filter");
+ if (ctx->HasOutput(framework::GradVarName("Input"))) {
+ ctx->SetOutputDim(framework::GradVarName("Input"), in_dims);
}
-};
+ if (ctx->HasOutput(framework::GradVarName("Filter"))) {
+ ctx->SetOutputDim(framework::GradVarName("Filter"), filter_dims);
+ }
+}
} // namespace operators
} // namespace paddle
diff --git a/paddle/operators/conv2d_op.cu b/paddle/operators/conv2d_op.cu
index 5df818ba04..c697c9466d 100644
--- a/paddle/operators/conv2d_op.cu
+++ b/paddle/operators/conv2d_op.cu
@@ -12,7 +12,7 @@
See the License for the specific language governing permissions and
limitations under the License. */
-#include "paddle/operators/gemm_conv2d_op.h"
+#include "paddle/operators/conv2d_op.h"
namespace ops = paddle::operators;
diff --git a/paddle/operators/gemm_conv2d_op.h b/paddle/operators/conv2d_op.h
similarity index 90%
rename from paddle/operators/gemm_conv2d_op.h
rename to paddle/operators/conv2d_op.h
index 323e3f7c3b..7ebdbe81cb 100644
--- a/paddle/operators/gemm_conv2d_op.h
+++ b/paddle/operators/conv2d_op.h
@@ -24,6 +24,38 @@ namespace operators {
using Tensor = framework::Tensor;
+// Base convolution operator definations for other conv
+// like operators to reuse the implementation.
+inline int OutputSize(int input_size, int filter_size, int padding,
+ int stride) {
+ int output_size = (input_size - filter_size + 2 * padding) / stride + 1;
+ return output_size;
+}
+
+// Define Op classes in .h file so that other conv
+// operator implementations can reuse the code.
+class Conv2DOpMaker : public framework::OpProtoAndCheckerMaker {
+ public:
+ Conv2DOpMaker(framework::OpProto* proto,
+ framework::OpAttrChecker* op_checker);
+};
+
+class Conv2DOp : public framework::OperatorWithKernel {
+ public:
+ using framework::OperatorWithKernel::OperatorWithKernel;
+
+ protected:
+ void InferShape(framework::InferShapeContext* ctx) const override;
+};
+
+class Conv2DOpGrad : public framework::OperatorWithKernel {
+ public:
+ using framework::OperatorWithKernel::OperatorWithKernel;
+
+ protected:
+ void InferShape(framework::InferShapeContext* ctx) const override;
+};
+
template <typename Place, typename T>
class GemmConv2DKernel : public framework::OpKernel<T> {
public:
@@ -74,7 +106,6 @@ class GemmConv2DKernel : public framework::OpKernel<T> {
framework::DDim output_matrix_shape = {output_channels,
output_height * output_width};
-
// convolution operator: im2col + gemm
int in_step = input_channels / groups;
int out_step = output_channels / groups;
diff --git a/paddle/operators/conv_cudnn_op.cc b/paddle/operators/conv_cudnn_op.cc
new file mode 100644
index 0000000000..4288f300dd
--- /dev/null
+++ b/paddle/operators/conv_cudnn_op.cc
@@ -0,0 +1,47 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. */
+
+#include "paddle/operators/conv2d_op.h"
+
+namespace paddle {
+namespace operators {
+
+class CudnnConvOpMaker : public Conv2DOpMaker {
+ public:
+ CudnnConvOpMaker(framework::OpProto* proto,
+ framework::OpAttrChecker* op_checker)
+ : Conv2DOpMaker(proto, op_checker) {
+ AddAttr<std::vector<int>>("dilations", "dilations of convolution operator.")
+ .SetDefault(std::vector<int>{1, 1});
+ AddAttr<int>("workspace_size_MB",
+ "workspace size for cudnn, in MB, "
+ "workspace is a section of GPU memory which will be "
+ "allocated/freed each time the operator runs, larger "
+ "workspace size can increase performance but also requires "
+ "better hardward. This size should be carefully setted.")
+ .SetDefault(4096);
+ }
+};
+
+} // namespace operators
+} // namespace paddle
+
+namespace ops = paddle::operators;
+REGISTER_OP(conv_cudnn, ops::Conv2DOp, ops::CudnnConvOpMaker, conv_cudnn_grad,
+ ops::Conv2DOpGrad);
+REGISTER_OP_CPU_KERNEL(
+ conv_cudnn, ops::GemmConv2DKernel<paddle::platform::CPUPlace, float>);
+REGISTER_OP_CPU_KERNEL(
+ conv_cudnn_grad,
+ ops::GemmConvGrad2DKernel<paddle::platform::CPUPlace, float>);
diff --git a/paddle/operators/conv_cudnn_op.cu b/paddle/operators/conv_cudnn_op.cu
new file mode 100644
index 0000000000..366d0323b8
--- /dev/null
+++ b/paddle/operators/conv_cudnn_op.cu
@@ -0,0 +1,277 @@
+/* Copyright (c) 2016 PaddlePaddle Authors All Rights Reserve.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. */
+
+#include "paddle/framework/eigen.h"
+#include "paddle/framework/op_registry.h"
+#include "paddle/memory/memory.h"
+#include "paddle/operators/conv2d_op.h"
+#include "paddle/platform/assert.h"
+#include "paddle/platform/cudnn_helper.h"
+
+namespace paddle {
+namespace operators {
+
+using Tensor = framework::Tensor;
+using ScopedTensorDescriptor = platform::ScopedTensorDescriptor;
+using ScopedFilterDescriptor = platform::ScopedFilterDescriptor;
+using ScopedConvolutionDescriptor = platform::ScopedConvolutionDescriptor;
+using DataLayout = platform::DataLayout;
+using CUDADeviceContext = platform::CUDADeviceContext;
+
+static constexpr size_t kCONV_CUDNN_WORKSPACE_LIMIT_BYTES = 1024 * 1024 * 1024;
+
+// NOTE: framework::vectorize converts to type int64_t
+// which does not fit cudnn inputs.
+std::vector<int> Dims2Vector(const framework::DDim& dims) {
+ std::vector<int> ret;
+ for (int i = 0; i < dims.size(); i++) {
+ ret.push_back(dims[i]);
+ }
+ return ret;
+}
+
+template <typename T>
+class CudnnConvOpKernel : public framework::OpKernel<T> {
+ public:
+ void Compute(const framework::ExecutionContext& ctx) const override {
+ PADDLE_ENFORCE(platform::is_gpu_place(ctx.GetPlace()),
+ "It must use GPUPlace.");
+ auto* input = ctx.Input<Tensor>("Input");
+ auto* filter = ctx.Input<Tensor>("Filter");
+ auto* output = ctx.Output<Tensor>("Output");
+
+ std::vector<int> strides = ctx.Attr<std::vector<int>>("strides");
+ std::vector<int> paddings = ctx.Attr<std::vector<int>>("paddings");
+ std::vector<int> dilations = ctx.Attr<std::vector<int>>("dilations");
+ int groups = ctx.Attr<int>("groups");
+ int user_workspace_size = ctx.Attr<int>("workspace_size_MB");
+
+ const T* input_data = input->data<T>();
+ const T* filter_data = filter->data<T>();
+ T* output_data = output->mutable_data<T>(ctx.GetPlace());
+
+ // ------------------- cudnn descriptors ---------------------
+ ScopedTensorDescriptor input_desc;
+ ScopedTensorDescriptor output_desc;
+ ScopedFilterDescriptor filter_desc;
+ ScopedConvolutionDescriptor conv_desc;
+ DataLayout layout = DataLayout::kNCHW;
+
+ cudnnTensorDescriptor_t cudnn_input_desc =
+ input_desc.descriptor<T>(layout, Dims2Vector(input->dims()), groups);
+ cudnnTensorDescriptor_t cudnn_output_desc =
+ output_desc.descriptor<T>(layout, Dims2Vector(output->dims()), groups);
+ cudnnFilterDescriptor_t cudnn_filter_desc =
+ filter_desc.descriptor<T>(layout, Dims2Vector(filter->dims()), groups);
+ cudnnConvolutionDescriptor_t cudnn_conv_desc =
+ conv_desc.descriptor<T>(paddings, strides, dilations);
+
+ int input_channels = input->dims()[1];
+ int input_height = input->dims()[2];
+ int input_width = input->dims()[3];
+ int output_channels = output->dims()[1];
+ int output_height = output->dims()[2];
+ int output_width = output->dims()[3];
+
+ int group_offset_in = input_channels / groups * input_height * input_width;
+ int group_offset_out =
+ output_channels / groups * output_height * output_width;
+ int group_offset_filter = filter->numel() / groups;
+ // ------------------- cudnn conv workspace ---------------------
+ void* cudnn_workspace = nullptr;
+ size_t workspace_size_in_bytes; // final workspace to allocate.
+ size_t workspace_size_limit = kCONV_CUDNN_WORKSPACE_LIMIT_BYTES;
+ if (user_workspace_size > 0) {
+ workspace_size_limit = user_workspace_size * 1024 * 1024;
+ }
+ // ------------------- cudnn conv algorithm ---------------------
+ cudnnConvolutionFwdAlgo_t algo;
+ auto handle = ctx.cuda_device_context().cudnn_handle();
+
+ PADDLE_ENFORCE(platform::dynload::cudnnGetConvolutionForwardAlgorithm(
+ handle, cudnn_input_desc, cudnn_filter_desc, cudnn_conv_desc,
+ cudnn_output_desc, CUDNN_CONVOLUTION_FWD_SPECIFY_WORKSPACE_LIMIT,
+ workspace_size_limit, &algo));
+ // get workspace size able to allocate
+ PADDLE_ENFORCE(platform::dynload::cudnnGetConvolutionForwardWorkspaceSize(
+ handle, cudnn_input_desc, cudnn_filter_desc, cudnn_conv_desc,
+ cudnn_output_desc, algo, &workspace_size_in_bytes));
+ // Allocate on GPU memory
+ platform::GPUPlace gpu = boost::get<platform::GPUPlace>(ctx.GetPlace());
+ cudnn_workspace = paddle::memory::Alloc(gpu, workspace_size_in_bytes);
+ // ------------------- cudnn conv forward ---------------------
+ T alpha = 1.0f, beta = 0.0f;
+ for (int i = 0; i < groups; i++) {
+ PADDLE_ENFORCE(platform::dynload::cudnnConvolutionForward(
+ handle, &alpha, cudnn_input_desc, input_data + i * group_offset_in,
+ cudnn_filter_desc, filter_data + i * group_offset_filter,
+ cudnn_conv_desc, algo, cudnn_workspace, workspace_size_in_bytes,
+ &beta, cudnn_output_desc, output_data + i * group_offset_out));
+ }
+ // Release the cudnn workspace
+ paddle::memory::Free(gpu, cudnn_workspace);
+ }
+};
+
+template <typename T>
+class CudnnConvGradOpKernel : public framework::OpKernel<T> {
+ public:
+ void Compute(const framework::ExecutionContext& ctx) const override {
+ PADDLE_ENFORCE(platform::is_gpu_place(ctx.GetPlace()),
+ "It must use GPUPlace.");
+ auto input = ctx.Input<Tensor>("Input");
+ auto filter = ctx.Input<Tensor>("Filter");
+ auto output_grad = ctx.Input<Tensor>(framework::GradVarName("Output"));
+ auto input_grad = ctx.Output<Tensor>(framework::GradVarName("Input"));
+ auto filter_grad = ctx.Output<Tensor>(framework::GradVarName("Filter"));
+
+ const T* input_data = input->data<T>();
+ const T* output_grad_data = output_grad->data<T>();
+ const T* filter_data = filter->data<T>();
+
+ std::vector<int> strides = ctx.Attr<std::vector<int>>("strides");
+ std::vector<int> paddings = ctx.Attr<std::vector<int>>("paddings");
+ std::vector<int> dilations = ctx.Attr<std::vector<int>>("dilations");
+ int groups = ctx.Attr<int>("groups");
+ int user_workspace_size = ctx.Attr<int>("workspace_size_MB");
+
+ // ------------------- cudnn descriptors ---------------------
+ ScopedTensorDescriptor input_desc;
+ ScopedTensorDescriptor output_grad_desc;
+ ScopedTensorDescriptor input_grad_desc;
+
+ ScopedFilterDescriptor filter_desc;
+ ScopedFilterDescriptor filter_grad_desc;
+ ScopedConvolutionDescriptor conv_desc;
+ DataLayout layout = DataLayout::kNCHW;
+
+ cudnnTensorDescriptor_t cudnn_input_desc =
+ input_desc.descriptor<T>(layout, Dims2Vector(input->dims()), groups);
+ cudnnTensorDescriptor_t cudnn_output_grad_desc =
+ output_grad_desc.descriptor<T>(layout, Dims2Vector(output_grad->dims()),
+ groups);
+ cudnnFilterDescriptor_t cudnn_filter_desc =
+ filter_desc.descriptor<T>(layout, Dims2Vector(filter->dims()), groups);
+ cudnnTensorDescriptor_t cudnn_input_grad_desc = nullptr;
+ cudnnFilterDescriptor_t cudnn_filter_grad_desc = nullptr;
+
+ cudnnConvolutionDescriptor_t cudnn_conv_desc =
+ conv_desc.descriptor<T>(paddings, strides, dilations);
+
+ int input_channels = input->dims()[1];
+ int input_height = input->dims()[2];
+ int input_width = input->dims()[3];
+ int output_grad_channels = filter->dims()[0];
+ int output_grad_height = output_grad->dims()[2];
+ int output_grad_width = output_grad->dims()[3];
+
+ int group_offset_in = input_channels / groups * input_height * input_width;
+ int group_offset_out =
+ output_grad_channels / groups * output_grad_height * output_grad_width;
+ int group_offset_filter = filter->numel() / groups;
+ // ------------------- cudnn backward algorithm ---------------------
+ cudnnConvolutionBwdDataAlgo_t data_algo;
+ cudnnConvolutionBwdFilterAlgo_t filter_algo;
+ size_t workspace_size_in_bytes = 0, tmp_size = 0;
+ size_t workspace_size_limit = kCONV_CUDNN_WORKSPACE_LIMIT_BYTES;
+ if (user_workspace_size > 0) {
+ workspace_size_limit = user_workspace_size * 1024 * 1024;
+ }
+
+ auto handle = ctx.cuda_device_context().cudnn_handle();
+ if (input_grad) {
+ cudnn_input_grad_desc = input_grad_desc.descriptor<T>(
+ layout, Dims2Vector(input_grad->dims()), groups);
+ PADDLE_ENFORCE(
+ platform::dynload::cudnnGetConvolutionBackwardDataAlgorithm(
+ handle, cudnn_filter_desc,
+ // dyDesc: Handle to the previously initialized input differential
+ // tensor descriptor.
+ cudnn_output_grad_desc, cudnn_conv_desc,
+ // dxDesc: Handle to the previously initialized output tensor
+ // descriptor.
+ cudnn_input_grad_desc,
+ CUDNN_CONVOLUTION_BWD_DATA_SPECIFY_WORKSPACE_LIMIT,
+ workspace_size_limit, &data_algo));
+ PADDLE_ENFORCE(
+ platform::dynload::cudnnGetConvolutionBackwardDataWorkspaceSize(
+ handle, cudnn_filter_desc, cudnn_output_grad_desc,
+ cudnn_conv_desc, cudnn_input_grad_desc, data_algo, &tmp_size));
+ workspace_size_in_bytes = std::max(workspace_size_in_bytes, tmp_size);
+ }
+
+ if (filter_grad) {
+ cudnn_filter_grad_desc = filter_grad_desc.descriptor<T>(
+ layout, Dims2Vector(filter_grad->dims()), groups);
+ PADDLE_ENFORCE(
+ platform::dynload::cudnnGetConvolutionBackwardFilterAlgorithm(
+ handle, cudnn_input_desc, cudnn_output_grad_desc, cudnn_conv_desc,
+ cudnn_filter_desc,
+ CUDNN_CONVOLUTION_BWD_FILTER_SPECIFY_WORKSPACE_LIMIT,
+ workspace_size_limit, &filter_algo));
+
+ PADDLE_ENFORCE(
+ platform::dynload::cudnnGetConvolutionBackwardFilterWorkspaceSize(
+ handle, cudnn_input_desc, cudnn_output_grad_desc, cudnn_conv_desc,
+ cudnn_filter_desc, filter_algo, &tmp_size));
+ workspace_size_in_bytes = std::max(workspace_size_in_bytes, tmp_size);
+ }
+ // ------------------- cudnn conv workspace ---------------------
+ // Already on GPU
+ void* cudnn_workspace = nullptr;
+ platform::GPUPlace gpu = boost::get<platform::GPUPlace>(ctx.GetPlace());
+ cudnn_workspace = paddle::memory::Alloc(gpu, workspace_size_in_bytes);
+ // ------------------- cudnn conv backward data ---------------------
+ // FIXME(typhoonzero): template type T may not be the same as cudnn call.
+ T alpha = 1.0f, beta = 0.0f;
+ if (input_grad) {
+ T* input_grad_data = input_grad->mutable_data<T>(ctx.GetPlace());
+ auto t = framework::EigenVector<T>::Flatten(*input_grad);
+ t.device(ctx.GetEigenDevice<platform::GPUPlace>()) =
+ t.constant(static_cast<T>(0));
+ for (int i = 0; i < groups; i++) {
+ PADDLE_ENFORCE(platform::dynload::cudnnConvolutionBackwardData(
+ handle, &alpha, cudnn_filter_desc,
+ filter_data + i * group_offset_filter, cudnn_output_grad_desc,
+ output_grad_data + i * group_offset_out, cudnn_conv_desc, data_algo,
+ cudnn_workspace, workspace_size_in_bytes, &beta,
+ cudnn_input_grad_desc, input_grad_data + i * group_offset_in));
+ }
+ }
+ // ------------------- cudnn conv backward filter ---------------------
+ if (filter_grad) {
+ T* filter_grad_data = filter_grad->mutable_data<T>(ctx.GetPlace());
+ auto t = framework::EigenVector<T>::Flatten(*filter_grad);
+ t.device(ctx.GetEigenDevice<platform::GPUPlace>()) =
+ t.constant(static_cast<T>(0));
+ for (int i = 0; i < groups; i++) {
+ PADDLE_ENFORCE(platform::dynload::cudnnConvolutionBackwardFilter(
+ handle, &alpha, cudnn_input_desc, input_data + i * group_offset_in,
+ cudnn_output_grad_desc, output_grad_data + i * group_offset_out,
+ cudnn_conv_desc, filter_algo, cudnn_workspace,
+ workspace_size_in_bytes, &beta, cudnn_filter_grad_desc,
+ filter_grad_data + i * group_offset_filter));
+ }
+ }
+ // Release the cudnn workspace
+ paddle::memory::Free(gpu, cudnn_workspace);
+ }
+};
+
+} // namespace operators
+} // namespace paddle
+
+REGISTER_OP_GPU_KERNEL(conv_cudnn, paddle::operators::CudnnConvOpKernel<float>);
+REGISTER_OP_GPU_KERNEL(conv_cudnn_grad,
+ paddle::operators::CudnnConvGradOpKernel<float>);
diff --git a/paddle/operators/decayed_adagrad_op.cc b/paddle/operators/decayed_adagrad_op.cc
new file mode 100644
index 0000000000..7f583f18c8
--- /dev/null
+++ b/paddle/operators/decayed_adagrad_op.cc
@@ -0,0 +1,96 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/operators/decayed_adagrad_op.h"
+
+namespace paddle {
+namespace operators {
+
+class DecayedAdagradOp : public framework::OperatorWithKernel {
+ public:
+ using framework::OperatorWithKernel::OperatorWithKernel;
+
+ protected:
+ void InferShape(framework::InferShapeContext *ctx) const override {
+ PADDLE_ENFORCE(ctx->HasInput("Param"),
+ "Input(Param) of DecayedAdagradOp should not be null.");
+ PADDLE_ENFORCE(ctx->HasInput("Grad"),
+ "Input(Grad) of DecayedAdagradOp should not be null.");
+ PADDLE_ENFORCE(ctx->HasInput("Moment"),
+ "Input(Moment) of DecayedAdagradOp should not be null.");
+ PADDLE_ENFORCE(
+ ctx->HasInput("LearningRate"),
+ "Input(LearningRate) of DecayedAdagradOp should not be null.");
+
+ PADDLE_ENFORCE(ctx->HasOutput("ParamOut"),
+ "Output(ParamOut) of DecayedAdagradOp should not be null.");
+ PADDLE_ENFORCE(ctx->HasOutput("MomentOut"),
+ "Output(MomentOut) of DecayedAdagradOp should not be null.");
+
+ auto lr_dims = ctx->GetInputDim("LearningRate");
+ PADDLE_ENFORCE_EQ(framework::product(lr_dims), 1,
+ "LearningRate should have one element");
+ auto param_dims = ctx->GetInputDim("Param");
+ PADDLE_ENFORCE_EQ(param_dims, ctx->GetInputDim("Grad"),
+ "Param and Grad input of DecayedAdagradOp should have "
+ "the same dimension.");
+ PADDLE_ENFORCE_EQ(param_dims, ctx->GetInputDim("Moment"),
+ "Param and Moment input of DecayedAdagradOp should have "
+ "the same dimension.");
+
+ ctx->SetOutputDim("ParamOut", param_dims);
+ ctx->SetOutputDim("MomentOut", param_dims);
+ }
+};
+
+class DecayedAdagradOpMaker : public framework::OpProtoAndCheckerMaker {
+ public:
+ DecayedAdagradOpMaker(framework::OpProto *proto,
+ framework::OpAttrChecker *op_checker)
+ : OpProtoAndCheckerMaker(proto, op_checker) {
+ AddInput("Param", "(Tensor) Input parameter");
+ AddInput("Grad", "(Tensor) Input gradient");
+ AddInput("Moment", "(Tensor) Second moment");
+ AddInput("LearningRate", "(Tensor) Learning rate");
+
+ AddOutput("ParamOut", "(Tensor) Output parameter");
+ AddOutput("MomentOut", "(Tensor) Output second moment");
+
+ AddAttr<float>("decay",
+ "(float, default 0.95) "
+ "Discounting factor for coming gradient")
+ .SetDefault(0.95);
+ AddAttr<float>("epsilon",
+ "(float, default 1.0e-6) "
+ "Constant for numerical stability")
+ .SetDefault(1.0e-6f);
+ AddComment(R"DOC(
+
+Decayed Adagrad
+
+moment_out = decay * moment + (1 - decay) * grad * grad
+param_out = param - learning_rate * grad / (sqrt(moment_out) + epsilon)
+
+)DOC");
+ }
+};
+} // namespace operators
+} // namespace paddle
+
+namespace ops = paddle::operators;
+REGISTER_OP_WITHOUT_GRADIENT(decayed_adagrad, ops::DecayedAdagradOp,
+ ops::DecayedAdagradOpMaker);
+REGISTER_OP_CPU_KERNEL(
+ decayed_adagrad,
+ ops::DecayedAdagradOpKernel<paddle::platform::CPUPlace, float>);
diff --git a/paddle/operators/decayed_adagrad_op.cu b/paddle/operators/decayed_adagrad_op.cu
new file mode 100644
index 0000000000..6fce77fe4e
--- /dev/null
+++ b/paddle/operators/decayed_adagrad_op.cu
@@ -0,0 +1,21 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. */
+
+#define EIGEN_USE_GPU
+#include "paddle/operators/decayed_adagrad_op.h"
+
+namespace ops = paddle::operators;
+REGISTER_OP_GPU_KERNEL(
+ decayed_adagrad,
+ ops::DecayedAdagradOpKernel<paddle::platform::GPUPlace, float>);
diff --git a/paddle/operators/decayed_adagrad_op.h b/paddle/operators/decayed_adagrad_op.h
new file mode 100644
index 0000000000..0fe0fc5acd
--- /dev/null
+++ b/paddle/operators/decayed_adagrad_op.h
@@ -0,0 +1,56 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+#include "paddle/framework/eigen.h"
+#include "paddle/framework/op_registry.h"
+
+namespace paddle {
+namespace operators {
+
+template <typename Place, typename T>
+class DecayedAdagradOpKernel : public framework::OpKernel<T> {
+ public:
+ void Compute(const framework::ExecutionContext& ctx) const override {
+ auto param_out_tensor = ctx.Output<framework::Tensor>("ParamOut");
+ auto moment_out_tensor = ctx.Output<framework::Tensor>("MomentOut");
+
+ param_out_tensor->mutable_data<T>(ctx.GetPlace());
+ moment_out_tensor->mutable_data<T>(ctx.GetPlace());
+
+ float decay = ctx.Attr<float>("decay");
+ float epsilon = ctx.Attr<float>("epsilon");
+
+ auto param = framework::EigenVector<T>::Flatten(
+ *ctx.Input<framework::Tensor>("Param"));
+ auto grad = framework::EigenVector<T>::Flatten(
+ *ctx.Input<framework::Tensor>("Grad"));
+ auto moment = framework::EigenVector<T>::Flatten(
+ *ctx.Input<framework::Tensor>("Moment"));
+ auto lr = framework::EigenVector<T>::Flatten(
+ *ctx.Input<framework::Tensor>("LearningRate"));
+
+ auto param_out = framework::EigenVector<T>::Flatten(*param_out_tensor);
+ auto moment_out = framework::EigenVector<T>::Flatten(*moment_out_tensor);
+ auto place = ctx.GetEigenDevice<Place>();
+
+ moment_out.device(place) = decay * moment + (1 - decay) * grad * grad;
+ Eigen::DSizes<int, 1> m_dsize(moment_out_tensor->numel());
+ param_out.device(place) =
+ param - lr.broadcast(m_dsize) * grad / (moment_out.sqrt() + epsilon);
+ }
+};
+
+} // namespace operators
+} // namespace paddle
diff --git a/paddle/operators/dynamic_recurrent_op.cc b/paddle/operators/dynamic_recurrent_op.cc
index b919aef8fb..58a5bf3e36 100644
--- a/paddle/operators/dynamic_recurrent_op.cc
+++ b/paddle/operators/dynamic_recurrent_op.cc
@@ -23,6 +23,7 @@ using framework::Scope;
using framework::TensorArray;
using framework::LoDTensor;
using framework::Variable;
+using framework::DySeqMetaBatch;
namespace detail {
@@ -33,6 +34,29 @@ inline void CreateVariables(Scope& scope,
}
}
+/*
+ * The inputs with sequence should be reordered when they are split, so the
+ * boot_states should be reordered in the same order.
+ *
+ * NOTE This may require that the `pre_state` of the first time step should just
+ * copy the `boot_state` rather than reference it, for that the content should
+ * be reordered, but the RNN op should not change the `boot_state` as an input
+ * variable's content.
+ */
+template <typename T>
+inline void ReorderBootState(const DySeqMetaBatch& metas,
+ const LoDTensor& boot_state, LoDTensor* tensor,
+ const platform::Place& dst_place) {
+ for (size_t seq_id = 0; seq_id < metas.size(); seq_id++) {
+ auto slice = tensor->Slice<T>(seq_id, seq_id + 1);
+ auto boot_slice =
+ boot_state.Slice<T>(metas[seq_id].ori_idx, metas[seq_id].ori_idx + 1);
+ // TODO(superjom) pass in device context as an argument
+ slice.template CopyFrom<T>(boot_slice, dst_place,
+ platform::CPUDeviceContext());
+ }
+}
+
} // namespace detail
class DynamicRecurrentOpProtoAndCheckerMaker
@@ -69,6 +93,7 @@ void DynamicRecurrentOp::Run(const Scope& scope,
CreateScopes();
WriteStepInputs();
InitStates();
+ WriteStepOutputs();
// call stepnet in all the time steps
for (size_t step = 0; step < cache_.num_steps; step++) {
@@ -76,7 +101,6 @@ void DynamicRecurrentOp::Run(const Scope& scope,
stepnet_->Run(step_scope, dev_ctx);
}
- WriteStepOutputs();
ConcatOutputs();
}
@@ -84,11 +108,11 @@ void DynamicRecurrentOp::SplitInputs() const {
// TODO(superjom) make level a config
// TODO(superjom) check all the inputs has the same LoD
int level = 0;
- const auto& inlinks = cache_.inlinks;
- for (const auto& item : inlinks) {
+ for (const auto& item : cache_.inlinks) {
const auto& var = item.second;
const auto& tensor = var->Get<LoDTensor>();
TensorArray& ta = step_inputs_[item.first];
+
dy_seq_metas_[item.first] =
ta.Unpack(tensor, level, true /*length_descend*/);
@@ -120,17 +144,11 @@ void DynamicRecurrentOp::WriteStepInputs() const {
}
void DynamicRecurrentOp::WriteStepOutputs() const {
- for (size_t step = 0; step < cache_.scopes->size(); step++) {
- auto& scope = cache_.GetScope(step);
- for (auto& item : step_outputs_) {
- auto* var = scope.FindVar(item.first);
- if (var == nullptr) {
- var = scope.NewVar(item.first);
- }
- auto* tensor = var->GetMutable<LoDTensor>();
- item.second.WriteShared(step, *tensor);
- }
+ // initialize step outputs
+ for (const auto& item : cache_.outlinks) {
+ step_outputs_.emplace(item.first, TensorArray());
}
+ PADDLE_ENFORCE_GT(step_outputs_.size(), 0UL);
}
void DynamicRecurrentOp::CreateScopes() const {
@@ -145,12 +163,18 @@ void DynamicRecurrentOp::CreateScopes() const {
PADDLE_ENFORCE_NOT_NULL(stepnet_, "stepnet should be set first");
std::vector<std::string> memories;
std::vector<std::string> pre_memories;
+ std::vector<std::string> stepnet_outputs;
std::transform(arg_.memories.begin(), arg_.memories.end(),
std::back_inserter(memories),
[](const rnn::MemoryAttr& m) { return m.var; });
std::transform(arg_.memories.begin(), arg_.memories.end(),
std::back_inserter(pre_memories),
[](const rnn::MemoryAttr& m) { return m.pre_var; });
+ for (const auto& item : stepnet_->Outputs()) {
+ for (const auto& var : item.second) {
+ stepnet_outputs.push_back(var);
+ }
+ }
for (size_t step = 0; step < cache_.num_steps; step++) {
auto& scope = cache_.GetScope(step);
@@ -158,60 +182,88 @@ void DynamicRecurrentOp::CreateScopes() const {
detail::CreateVariables(scope, arg_.outlinks);
detail::CreateVariables(scope, memories);
detail::CreateVariables(scope, pre_memories);
+ detail::CreateVariables(scope, stepnet_outputs);
}
}
void DynamicRecurrentOp::ConcatOutputs() const {
// TODO(superjom) transform this to a config
int level = 0;
- // TODO(superjom) pass in some lod
- // just a placeholder
- framework::LoD lod;
+ for (size_t step = 0; step < cache_.num_steps; step++) {
+ auto& scope = cache_.GetScope(step);
+ for (auto& item : step_outputs_) {
+ auto* var = scope.FindVar(item.first);
+ PADDLE_ENFORCE_NOT_NULL(var);
+ auto* tensor = var->GetMutable<LoDTensor>();
+ tensor->mutable_data<value_type>(platform::CPUPlace());
+ item.second.WriteShared(step, *tensor);
+ }
+ }
+ // the inlinks' lods should be the same, so randomly get one lod.
+ const auto& some_lod =
+ cache_.scope->FindVar(arg_.inlinks.front())->Get<LoDTensor>().lod();
+ const auto& some_meta = dy_seq_metas_[arg_.inlinks.front()];
for (auto& item : step_outputs_) {
- auto tensor = item.second.Pack(level, dy_seq_metas_[item.first], lod);
- auto& output = cache_.outlinks[item.first]->Get<LoDTensor>();
- const_cast<LoDTensor*>(&output)->ShareDataWith<value_type>(tensor);
+ auto tensor = item.second.Pack(level, some_meta, some_lod);
+ auto* output = cache_.outlinks[item.first]->GetMutable<LoDTensor>();
+ const_cast<LoDTensor*>(output)->ShareDataWith<value_type>(tensor);
}
}
void DynamicRecurrentOp::InitStates() const {
- // init the first state
- // TODO(superjom) parepare the scenerio that boot state not exists
- for (auto memory : arg_.memories) {
- auto* boot_state_var = cache_.scope->FindVar(memory.boot_var);
- PADDLE_ENFORCE_NOT_NULL(boot_state_var);
- auto& boot_state = boot_state_var->Get<LoDTensor>();
- const auto& dims = boot_state.dims();
-
- for (size_t step = 0; step < cache_.num_steps; step++) {
- auto& cur_scope = cache_.GetScope(step);
- // link pre-state to boot_state
- // init state and pre-state
- auto* pre_state = cur_scope.FindVar(memory.pre_var);
- PADDLE_ENFORCE_NOT_NULL(pre_state);
- pre_state->GetMutable<LoDTensor>();
-
- auto* state = cur_scope.FindVar(memory.var);
- PADDLE_ENFORCE_NOT_NULL(state);
- state->GetMutable<LoDTensor>()->Resize(dims);
- state->GetMutable<LoDTensor>()->mutable_data<value_type>(
- platform::CPUPlace());
-
- if (step == 0) {
- auto* pre_state_tensor = pre_state->GetMutable<LoDTensor>();
- pre_state_tensor->Resize(boot_state.dims());
- pre_state_tensor->ShareDataWith<value_type>(boot_state);
- } else {
- auto& pre_scope = cache_.GetScope(step - 1);
- auto* state_pre = pre_scope.FindVar(memory.var);
- PADDLE_ENFORCE_NOT_NULL(state_pre);
- pre_state->GetMutable<LoDTensor>()->ShareDataWith<value_type>(
- *state_pre->GetMutable<LoDTensor>());
- }
+ for (size_t step = 0; step < cache_.num_steps; step++) {
+ for (const auto& memory : arg_.memories) {
+ CreateState(memory, step);
+ LinkState(memory, step);
}
}
}
+void DynamicRecurrentOp::CreateState(const rnn::MemoryAttr& memory,
+ size_t step) const {
+ auto& scope = cache_.GetScope(step);
+ auto& state = *cache_.GetTensor(scope, memory.var);
+ auto& boot_state = *cache_.GetTensor(*cache_.scope, memory.boot_var);
+
+ size_t num_instances =
+ step_inputs_[arg_.inlinks.front()].Read(step).dims()[0];
+ auto dims = boot_state.dims();
+ dims[0] = num_instances;
+
+ state.Resize(dims);
+ state.mutable_data<value_type>(platform::CPUPlace());
+ states_[memory.var].WriteShared(step, state);
+}
+
+void DynamicRecurrentOp::LinkState(const rnn::MemoryAttr& memory,
+ size_t step) const {
+ auto& scope = cache_.GetScope(step);
+ auto& state_pre = *cache_.GetTensor(scope, memory.pre_var);
+
+ // all the step_inputs' metas should be the same, just randomly select one
+ // and get the dyseq meta.
+ const auto& some_meta = dy_seq_metas_[arg_.inlinks.front()];
+ size_t num_instances =
+ step_inputs_[arg_.inlinks.front()].Read(step).dims()[0];
+
+ LoDTensor* pre_state{nullptr};
+ if (step == 0) {
+ pre_state = cache_.GetTensor(*cache_.scope, memory.boot_var);
+ pre_state->mutable_data<float>(platform::CPUPlace());
+ // allocate memory
+ state_pre.Resize(pre_state->dims());
+ state_pre.mutable_data<value_type>(platform::CPUPlace());
+ detail::ReorderBootState<value_type>(some_meta, *pre_state, &state_pre,
+ pre_state->place());
+ } else {
+ pre_state = cache_.GetTensor(cache_.GetScope(step - 1), memory.var);
+ }
+
+ // shink and share from previous state
+ auto shrinked_pre_state = pre_state->Slice<value_type>(0, num_instances);
+ state_pre.ShareDataWith<value_type>(shrinked_pre_state);
+}
+
void DynamicRecurrentOp::ArgCache::Init(
const rnn::ArgumentName& name, const paddle::framework::OperatorBase& op,
const paddle::framework::Scope& scope, rnn::Argument* arg) {
@@ -261,6 +313,12 @@ Variable* DynamicRecurrentOp::ArgCache::GetVariable(const Scope& scope,
return var;
}
+LoDTensor* DynamicRecurrentOp::ArgCache::GetTensor(
+ const framework::Scope& scope, const std::string& name) {
+ auto* var = GetVariable(scope, name);
+ return var->GetMutable<LoDTensor>();
+}
+
const rnn::ArgumentName DynamicRecurrentOp::kArgName{
"step_net", "step_scopes", "inlinks", "outlinks",
"memories", "pre_memories", "boot_memories"};
diff --git a/paddle/operators/dynamic_recurrent_op.h b/paddle/operators/dynamic_recurrent_op.h
index 6a2970f27f..ec80a1c90e 100644
--- a/paddle/operators/dynamic_recurrent_op.h
+++ b/paddle/operators/dynamic_recurrent_op.h
@@ -77,6 +77,17 @@ class DynamicRecurrentOp : public framework::OperatorBase {
*/
void InitStates() const;
+ /*
+ * Create state variables for each time step.
+ */
+ void CreateState(const rnn::MemoryAttr& memory, size_t step) const;
+
+ /*
+ * Link pre-state variable in current scope to the state variable in the
+ * previous time step (scope).
+ */
+ void LinkState(const rnn::MemoryAttr& memory, size_t step) const;
+
/*
* Concatenate outputs in each time step and generate a LoDTensor.
*/
@@ -91,6 +102,16 @@ class DynamicRecurrentOp : public framework::OperatorBase {
}
const OperatorBase& GetStepNet() const { return *stepnet_; }
+ const framework::TensorArray& state(const std::string& name) const {
+ return states_[name];
+ }
+ const framework::TensorArray& step_input(const std::string& name) const {
+ return step_inputs_[name];
+ }
+ const framework::TensorArray& step_output(const std::string& name) const {
+ return step_outputs_[name];
+ }
+
protected:
struct ArgCache {
framework::Scope const* scope;
@@ -108,6 +129,9 @@ class DynamicRecurrentOp : public framework::OperatorBase {
return *scopes->at(index);
}
+ framework::LoDTensor* GetTensor(const framework::Scope& scope,
+ const std::string& name);
+
private:
void InitArgument(const rnn::ArgumentName& name, const OperatorBase& op,
rnn::Argument* arg);
@@ -122,7 +146,7 @@ class DynamicRecurrentOp : public framework::OperatorBase {
private:
std::unique_ptr<OperatorBase> stepnet_;
- mutable framework::TensorArray states_;
+ mutable std::map<std::string, framework::TensorArray> states_;
mutable std::map<std::string, framework::TensorArray> step_inputs_;
mutable std::map<std::string, framework::TensorArray> step_outputs_;
mutable std::map<std::string, std::vector<framework::DySeqMeta>>
diff --git a/paddle/operators/dynamic_recurrent_op_test.cc b/paddle/operators/dynamic_recurrent_op_test.cc
index 675a7890f3..b849c4541d 100644
--- a/paddle/operators/dynamic_recurrent_op_test.cc
+++ b/paddle/operators/dynamic_recurrent_op_test.cc
@@ -87,7 +87,6 @@ class DynamicRecurrentOpTestHelper : public ::testing::Test {
platform::CPUPlace place;
scope.NewVar("step_scopes");
CreateVar(scope, "boot_mem", framework::make_ddim({10, 20}), place);
- // auto* out0 =
CreateVar(scope, "out0", framework::make_ddim({10, 20}), place);
auto* in0 = CreateVar(scope, "in0", framework::make_ddim({10, 8}), place);
// 10 instanes with 4 sentences, length is 4, 3, 2, 1 respectively.
diff --git a/paddle/operators/feed_op.cc b/paddle/operators/feed_op.cc
new file mode 100644
index 0000000000..fa325bb282
--- /dev/null
+++ b/paddle/operators/feed_op.cc
@@ -0,0 +1,59 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/operators/feed_op.h"
+
+namespace paddle {
+namespace operators {
+
+class FeedOp : public framework::OperatorWithKernel {
+ public:
+ using framework::OperatorWithKernel::OperatorWithKernel;
+
+ protected:
+ void InferShape(framework::InferShapeContext* ctx) const override {
+ PADDLE_ENFORCE(ctx->HasOutput("Out"), "Output should be not null.");
+ auto& shape = ctx->Attrs().Get<std::vector<int>>("dims");
+ std::vector<int64_t> shape_int64(shape.size(), 0);
+ std::transform(shape.begin(), shape.end(), shape_int64.begin(),
+ [](int a) { return static_cast<int64_t>(a); });
+ ctx->SetOutputDim("Out", framework::make_ddim(shape_int64));
+ // TODO(qijun): need to handle LodTensor later
+ }
+
+ framework::DataType IndicateDataType(
+ const framework::ExecutionContext& ctx) const override {
+ return static_cast<framework::DataType>(Attr<int>("dataType"));
+ }
+};
+
+class FeedOpMaker : public framework::OpProtoAndCheckerMaker {
+ public:
+ FeedOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker)
+ : OpProtoAndCheckerMaker(proto, op_checker) {
+ AddAttr<int>("dataType", "output data type")
+ .SetDefault(framework::DataType::FP32);
+ AddAttr<int>("col", "The col in global feed variable").SetDefault(0);
+ AddAttr<std::vector<int>>("dims", "The dimension of feed tensor.");
+ AddOutput("Out", "The output of feed op.");
+ AddComment(R"DOC(Feed data from global feed variable)DOC");
+ }
+};
+
+} // namespace operators
+} // namespace paddle
+
+namespace ops = paddle::operators;
+REGISTER_OP_WITHOUT_GRADIENT(feed, ops::FeedOp, ops::FeedOpMaker);
+REGISTER_OP_CPU_KERNEL(feed, ops::FeedKernel<float>);
diff --git a/paddle/operators/feed_op.cu b/paddle/operators/feed_op.cu
new file mode 100644
index 0000000000..7b6a2ac91e
--- /dev/null
+++ b/paddle/operators/feed_op.cu
@@ -0,0 +1,18 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/operators/feed_op.h"
+
+namespace ops = paddle::operators;
+REGISTER_OP_GPU_KERNEL(feed, ops::FeedKernel<float>);
diff --git a/paddle/operators/feed_op.h b/paddle/operators/feed_op.h
new file mode 100644
index 0000000000..e756cd1842
--- /dev/null
+++ b/paddle/operators/feed_op.h
@@ -0,0 +1,42 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+#include "paddle/framework/eigen.h"
+#include "paddle/framework/op_registry.h"
+
+namespace paddle {
+namespace operators {
+
+template <typename T>
+class FeedKernel : public framework::OpKernel<T> {
+ public:
+ void Compute(const framework::ExecutionContext& ctx) const override {
+ framework::Tensor* out = ctx.Output<framework::Tensor>("Out");
+ out->mutable_data<T>(ctx.GetPlace());
+ framework::Variable* g_feed_variable =
+ framework::GetGlobalScope().FindVar("feed_value");
+ const auto& tensors =
+ g_feed_variable->Get<std::vector<framework::Tensor>>();
+ int col = ctx.template Attr<int>("col");
+ PADDLE_ENFORCE_GT(tensors.size(), static_cast<size_t>(col));
+ // TODO(qijun):
+ // check tensors[col].dims() with attribute,
+ // except the first dimenson.
+ out->CopyFrom<T>(tensors[col], ctx.GetPlace(), ctx.device_context());
+ }
+};
+
+} // namespace operators
+} // namespace paddle
diff --git a/paddle/operators/fetch_op.cc b/paddle/operators/fetch_op.cc
new file mode 100644
index 0000000000..90737c8c55
--- /dev/null
+++ b/paddle/operators/fetch_op.cc
@@ -0,0 +1,52 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/operators/fetch_op.h"
+
+namespace paddle {
+namespace operators {
+
+class FetchOp : public framework::OperatorWithKernel {
+ public:
+ using framework::OperatorWithKernel::OperatorWithKernel;
+
+ protected:
+ void InferShape(framework::InferShapeContext* ctx) const override {
+ PADDLE_ENFORCE(ctx->HasInput("Input"), "Input should be not null.");
+ }
+
+ framework::DataType IndicateDataType(
+ const framework::ExecutionContext& ctx) const override {
+ return static_cast<framework::DataType>(Attr<int>("dataType"));
+ }
+};
+
+class FetchOpMaker : public framework::OpProtoAndCheckerMaker {
+ public:
+ FetchOpMaker(framework::OpProto* proto, framework::OpAttrChecker* op_checker)
+ : OpProtoAndCheckerMaker(proto, op_checker) {
+ AddAttr<int>("dataType", "output data type")
+ .SetDefault(framework::DataType::FP32);
+ AddAttr<int>("col", "The col in global fetch variable").SetDefault(0);
+ AddInput("Input", "The output of fetch op.");
+ AddComment(R"DOC(Fetch data to global fetch variable)DOC");
+ }
+};
+
+} // namespace operators
+} // namespace paddle
+
+namespace ops = paddle::operators;
+REGISTER_OP_WITHOUT_GRADIENT(fetch, ops::FetchOp, ops::FetchOpMaker);
+REGISTER_OP_CPU_KERNEL(fetch, ops::FetchKernel<float>);
diff --git a/paddle/operators/fetch_op.cu b/paddle/operators/fetch_op.cu
new file mode 100644
index 0000000000..ca39d24c79
--- /dev/null
+++ b/paddle/operators/fetch_op.cu
@@ -0,0 +1,18 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/operators/fetch_op.h"
+
+namespace ops = paddle::operators;
+REGISTER_OP_GPU_KERNEL(fetch, ops::FetchKernel<float>);
diff --git a/paddle/operators/fetch_op.h b/paddle/operators/fetch_op.h
new file mode 100644
index 0000000000..b2a6e95875
--- /dev/null
+++ b/paddle/operators/fetch_op.h
@@ -0,0 +1,45 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+#include "paddle/framework/eigen.h"
+#include "paddle/framework/op_registry.h"
+
+namespace paddle {
+namespace operators {
+
+template <typename T>
+class FetchKernel : public framework::OpKernel<T> {
+ public:
+ void Compute(const framework::ExecutionContext& ctx) const override {
+ const framework::Tensor* input = ctx.Input<framework::Tensor>("Input");
+ framework::Variable* g_fetch_variable =
+ framework::GetGlobalScope().FindVar("fetch_value");
+ auto* tensors =
+ g_fetch_variable->GetMutable<std::vector<framework::Tensor>>();
+ int col = ctx.template Attr<int>("col");
+ if (tensors->size() < static_cast<size_t>(col + 1)) {
+ tensors->resize(col + 1);
+ }
+ PADDLE_ENFORCE_GT(tensors->size(), static_cast<size_t>(col));
+ (*tensors)[col].Resize(input->dims());
+ (*tensors)[col].mutable_data<T>(platform::CPUPlace());
+ (*tensors)[col].CopyFrom<T>(*input, platform::CPUPlace(),
+ ctx.device_context());
+ // TODO(qijun): need to handle LodTensor later
+ }
+};
+
+} // namespace operators
+} // namespace paddle
diff --git a/paddle/operators/margin_rank_loss_op.cc b/paddle/operators/margin_rank_loss_op.cc
new file mode 100644
index 0000000000..5be61dfec3
--- /dev/null
+++ b/paddle/operators/margin_rank_loss_op.cc
@@ -0,0 +1,124 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. */
+
+#include "paddle/operators/margin_rank_loss_op.h"
+
+namespace paddle {
+namespace operators {
+
+class MarginRankLossOp : public framework::OperatorWithKernel {
+ public:
+ using framework::OperatorWithKernel::OperatorWithKernel;
+
+ protected:
+ void InferShape(framework::InferShapeContext *ctx) const override {
+ // input check
+ PADDLE_ENFORCE(ctx->HasInput("Label"), "Input(Label) shouldn't be null.");
+ PADDLE_ENFORCE(ctx->HasInput("X1"), "Input(X1) shouldn't be null.");
+ PADDLE_ENFORCE(ctx->HasInput("X2"), "Input(X2) shouldn't be null.");
+ PADDLE_ENFORCE(ctx->HasOutput("Out"), "Output(Out) shouldn't be null.");
+ auto label_dims = ctx->GetInputDim("Label");
+ auto x1_dims = ctx->GetInputDim("X1");
+ auto x2_dims = ctx->GetInputDim("X2");
+ PADDLE_ENFORCE(
+ (label_dims == x1_dims) && (x1_dims == x2_dims) &&
+ (label_dims.size() == 2) && (label_dims[1] == 1),
+ "All inputs must be 2-D tensor with shape [batch_size x 1].");
+ ctx->SetOutputDim("Activated", label_dims);
+ ctx->SetOutputDim("Out", label_dims);
+ }
+};
+
+template <typename T>
+class MarginRankLossOpMaker : public framework::OpProtoAndCheckerMaker {
+ public:
+ MarginRankLossOpMaker(framework::OpProto *proto,
+ framework::OpAttrChecker *op_checker)
+ : OpProtoAndCheckerMaker(proto, op_checker) {
+ AddInput("X1",
+ "(2-D tensor with shape [batch_size x 1]) The score for "
+ "one item X1 to be ranked, from pairwise ranking model.");
+ AddInput("X2",
+ "(2-D tensor with shape [batch_size x 1]) The score for "
+ "another item X2 to be ranked, from pairwise ranking model.");
+ AddInput("Label",
+ "(2-D tensor with shape [batch_size x 1]) "
+ "The label indicating X1 ranked higher than X2 or not, "
+ "can only be +1 or -1.");
+ AddAttr<T>("margin", "(scalar, default 0) Margin for MarginRankLossOp.")
+ .SetDefault(static_cast<T>(0));
+ AddOutput("Activated",
+ "(2-D tensor with shape [batch_size x 1]) Intermediate tensor "
+ "to indicate whether each element of Output(Out) is activated.")
+ .AsIntermediate();
+ AddOutput("Out",
+ "(2-D tensor with shape [batch_size x 1]) "
+ "The output loss of MarginRankLoss operator.");
+ AddComment(R"DOC(
+
+MarginRankLoss operator measures the loss given a pair of training sample
+{`X1`, `X2`} and the `Label` with attribute `margin`, where `Label = +1`
+indicating X1 is ranked higher than `X2`, otherwise `Label = -1`. The loss
+turns out
+
+loss(X1, X2, Label) = max(0, -Label * (X1 - X2) + margin).
+
+The attribute `margin` involved here helps make the predictions more robust.
+Denote the item ranked higher as the positive sample, otherwise the negative
+sample. If the score of the two samples satisfies
+
+positive sample - negative sample < margin,
+
+the pair of samples will contribute to the final loss, which will backpropogate
+and train the ranking model to enlarge the difference of the two score.
+
+For batch input with size `batch_size`, `X1`, `X2` and `Label`
+all have the same shape [batch_size x 1].
+
+)DOC");
+ }
+};
+
+class MarginRankLossGradOp : public framework::OperatorWithKernel {
+ public:
+ using framework::OperatorWithKernel::OperatorWithKernel;
+
+ protected:
+ void InferShape(framework::InferShapeContext *ctx) const override {
+ PADDLE_ENFORCE(ctx->HasInput("Label"), "Input(Label) shouldn't be null.");
+ PADDLE_ENFORCE(ctx->HasInput("X1"), "Input(X1) shouldn't be null.");
+ PADDLE_ENFORCE(ctx->HasInput("X2"), "Input(X2) shouldn't be null.");
+ PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Out")),
+ "Input(Out@GRAD) shouldn't be null.");
+ PADDLE_ENFORCE(ctx->HasInput("Activated"),
+ "Intermediate(Activated) shouldn't be null.");
+ auto dims = ctx->GetInputDim("Label");
+ ctx->SetOutputDim(framework::GradVarName("X1"), dims);
+ ctx->SetOutputDim(framework::GradVarName("X2"), dims);
+ }
+};
+
+} // namespace operators
+} // namespace paddle
+namespace ops = paddle::operators;
+
+REGISTER_OP(margin_rank_loss, ops::MarginRankLossOp,
+ ops::MarginRankLossOpMaker<float>, margin_rank_loss_grad,
+ ops::MarginRankLossGradOp);
+REGISTER_OP_CPU_KERNEL(
+ margin_rank_loss,
+ ops::MarginRankLossKernel<paddle::platform::CPUPlace, float>);
+REGISTER_OP_CPU_KERNEL(
+ margin_rank_loss_grad,
+ ops::MarginRankLossGradKernel<paddle::platform::CPUPlace, float>);
diff --git a/paddle/operators/margin_rank_loss_op.cu b/paddle/operators/margin_rank_loss_op.cu
new file mode 100644
index 0000000000..3a639f25d4
--- /dev/null
+++ b/paddle/operators/margin_rank_loss_op.cu
@@ -0,0 +1,24 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. */
+
+#include "paddle/operators/margin_rank_loss_op.h"
+
+namespace ops = paddle::operators;
+
+REGISTER_OP_GPU_KERNEL(
+ margin_rank_loss,
+ ops::MarginRankLossKernel<paddle::platform::GPUPlace, float>);
+REGISTER_OP_GPU_KERNEL(
+ margin_rank_loss_grad,
+ ops::MarginRankLossGradKernel<paddle::platform::GPUPlace, float>);
diff --git a/paddle/operators/margin_rank_loss_op.h b/paddle/operators/margin_rank_loss_op.h
new file mode 100644
index 0000000000..8d0830147e
--- /dev/null
+++ b/paddle/operators/margin_rank_loss_op.h
@@ -0,0 +1,98 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. */
+
+#pragma once
+
+#include "paddle/framework/eigen.h"
+#include "paddle/framework/op_registry.h"
+
+namespace paddle {
+namespace operators {
+
+template <typename T>
+struct ReLU {
+ HOSTDEVICE T operator()(const T& val) const {
+ return val > 0 ? val : static_cast<T>(0);
+ }
+};
+
+template <typename T>
+struct Heaviside {
+ HOSTDEVICE T operator()(const T& val) const {
+ return static_cast<T>(val > 0 ? 1 : 0);
+ }
+};
+
+template <typename Place, typename T>
+class MarginRankLossKernel : public framework::OpKernel<T> {
+ public:
+ void Compute(const framework::ExecutionContext& ctx) const {
+ auto* out_t = ctx.Output<framework::Tensor>("Out");
+ auto* act_t = ctx.Output<framework::Tensor>("Activated");
+
+ auto* label_t = ctx.Input<framework::Tensor>("Label");
+ auto* x1_t = ctx.Input<framework::Tensor>("X1");
+ auto* x2_t = ctx.Input<framework::Tensor>("X2");
+
+ out_t->mutable_data<T>(ctx.GetPlace());
+ act_t->mutable_data<T>(ctx.GetPlace());
+
+ auto margin = static_cast<T>(ctx.Attr<T>("margin"));
+ auto out = framework::EigenVector<T>::Flatten(*out_t);
+ auto act = framework::EigenVector<T>::Flatten(*act_t);
+
+ auto label = framework::EigenVector<T>::Flatten(*label_t);
+ auto x1 = framework::EigenVector<T>::Flatten(*x1_t);
+ auto x2 = framework::EigenVector<T>::Flatten(*x2_t);
+
+ auto& dev = ctx.GetEigenDevice<Place>();
+ out.device(dev) = (-label * (x1 - x2) + margin).unaryExpr(ReLU<T>());
+ act.device(dev) = out.unaryExpr(Heaviside<T>());
+ }
+};
+
+template <typename Place, typename T>
+class MarginRankLossGradKernel : public framework::OpKernel<T> {
+ public:
+ void Compute(const framework::ExecutionContext& ctx) const {
+ auto* d_x1_t =
+ ctx.Output<framework::LoDTensor>(framework::GradVarName("X1"));
+ auto* d_x2_t =
+ ctx.Output<framework::LoDTensor>(framework::GradVarName("X2"));
+
+ auto* act_t = ctx.Input<framework::Tensor>("Activated");
+ auto* d_out_t = ctx.Input<framework::Tensor>(framework::GradVarName("Out"));
+ auto* label_t = ctx.Input<framework::Tensor>("Label");
+
+ auto d_out = framework::EigenVector<T>::Flatten(*d_out_t);
+ auto act = framework::EigenVector<T>::Flatten(*act_t);
+ auto label = framework::EigenVector<T>::Flatten(*label_t);
+ auto& dev = ctx.GetEigenDevice<Place>();
+
+ // compute d_x1
+ if (d_x1_t) {
+ d_x1_t->mutable_data<T>(ctx.GetPlace());
+ auto d_x1 = framework::EigenVector<T>::Flatten(*d_x1_t);
+ d_x1.device(dev) = -d_out * act * label;
+ }
+ // compute d_x2
+ if (d_x2_t) {
+ d_x2_t->mutable_data<T>(ctx.GetPlace());
+ auto d_x2 = framework::EigenVector<T>::Flatten(*d_x2_t);
+ d_x2.device(dev) = d_out * act * label;
+ }
+ }
+};
+} // namespace operators
+} // namespace paddle
diff --git a/paddle/operators/math/CMakeLists.txt b/paddle/operators/math/CMakeLists.txt
index a0ceb029e3..1a2f623ce7 100644
--- a/paddle/operators/math/CMakeLists.txt
+++ b/paddle/operators/math/CMakeLists.txt
@@ -1,13 +1,18 @@
if(WITH_GPU)
- nv_library(math_function SRCS math_function.cc math_function.cu im2col.cc im2col.cu pooling.cc pooling.cu DEPS cblas device_context operator)
+ nv_library(math_function SRCS math_function.cc math_function.cu im2col.cc im2col.cu DEPS cblas device_context operator)
nv_test(math_function_test SRCS math_function_test.cc DEPS math_function tensor)
nv_library(softmax SRCS softmax.cc softmax.cu DEPS operator)
nv_library(cross_entropy SRCS cross_entropy.cc cross_entropy.cu DEPS operator)
+ nv_library(pooling SRCS pooling.cc pooling.cu DEPS device_context)
+ nv_library(vol2col SRCS vol2col.cc vol2col.cu DEPS device_context)
else()
- cc_library(math_function SRCS math_function.cc im2col.cc pooling.cc DEPS cblas device_context operator)
+ cc_library(math_function SRCS math_function.cc im2col.cc DEPS cblas device_context operator)
cc_test(math_function_test SRCS math_function_test.cc DEPS math_function tensor)
cc_library(softmax SRCS softmax.cc DEPS operator)
cc_library(cross_entropy SRCS cross_entropy.cc DEPS operator)
+ cc_library(pooling SRCS pooling.cc DEPS device_context)
+ cc_library(vol2col SRCS vol2col.cc DEPS device_context)
endif()
cc_test(im2col_test SRCS im2col_test.cc DEPS math_function tensor)
+cc_test(vol2col_test SRCS vol2col_test.cc DEPS vol2col tensor)
diff --git a/paddle/operators/math/im2col_test.cc b/paddle/operators/math/im2col_test.cc
index 40bdbfe733..9c506ae89b 100644
--- a/paddle/operators/math/im2col_test.cc
+++ b/paddle/operators/math/im2col_test.cc
@@ -49,10 +49,22 @@ void testIm2col() {
memcpy(input_ptr, arr, 6 * sizeof(float));
auto* place = new Place();
+ paddle::platform::DeviceContext* context;
+ if (paddle::platform::is_cpu_place(*place)) {
+ context =
+ new paddle::platform::CPUDeviceContext(paddle::platform::CPUPlace());
+ } else {
+#ifdef PADDLE_WITH_CUDA
+ context =
+ new paddle::platform::CUDADeviceContext(paddle::platform::GPUPlace());
+#else
+ PADDLE_THROW("no GPU support");
+#endif // PADDLE_ONLY_CPU
+ }
if (paddle::platform::is_cpu_place(*place)) {
input = input_tmp;
} else {
- input.CopyFrom<float>(input_tmp, *place);
+ input.CopyFrom<float>(input_tmp, *place, *context);
}
output_cfo.mutable_data<float>(
{1, filter_size, filter_size, output_height, output_width}, *place);
@@ -66,18 +78,6 @@ void testIm2col() {
paddle::operators::math::ColFormat::kOCF, Place, float>
im2col_ocf;
- paddle::platform::DeviceContext* context;
- if (paddle::platform::is_cpu_place(*place)) {
- context =
- new paddle::platform::CPUDeviceContext(paddle::platform::CPUPlace());
- } else {
-#ifdef PADDLE_WITH_CUDA
- context =
- new paddle::platform::CUDADeviceContext(paddle::platform::GPUPlace());
-#else
- PADDLE_THROW("no GPU support");
-#endif // PADDLE_ONLY_CPU
- }
im2col(*context, input, output_cfo, stride, stride, padding, padding);
im2col_ocf(*context, input, output_ocf, stride, stride, padding, padding);
@@ -85,7 +85,8 @@ void testIm2col() {
if (paddle::platform::is_cpu_place(*place)) {
out_cfo_ptr = output_cfo.data<float>();
} else {
- output_tmp.CopyFrom<float>(output_cfo, paddle::platform::CPUPlace());
+ output_tmp.CopyFrom<float>(output_cfo, paddle::platform::CPUPlace(),
+ *context);
out_cfo_ptr = output_tmp.data<float>();
}
EXPECT_EQ(out_cfo_ptr[0], 0);
@@ -101,7 +102,8 @@ void testIm2col() {
if (paddle::platform::is_cpu_place(*place)) {
out_ocf_ptr = output_ocf.data<float>();
} else {
- output_tmp.CopyFrom<float>(output_ocf, paddle::platform::CPUPlace());
+ output_tmp.CopyFrom<float>(output_ocf, paddle::platform::CPUPlace(),
+ *context);
out_ocf_ptr = output_tmp.data<float>();
}
EXPECT_EQ(out_ocf_ptr[0], 0);
diff --git a/paddle/operators/math/math_function_test.cc b/paddle/operators/math/math_function_test.cc
index 9945ba101d..c87d200c3a 100644
--- a/paddle/operators/math/math_function_test.cc
+++ b/paddle/operators/math/math_function_test.cc
@@ -17,17 +17,18 @@ TEST(math_function, notrans_mul_trans) {
auto* gpu_place = new paddle::platform::GPUPlace(0);
paddle::platform::CUDADeviceContext context(*gpu_place);
- input1_gpu.CopyFrom<float>(input1, *gpu_place);
- input2_gpu.CopyFrom<float>(input1, *gpu_place);
+ input1_gpu.CopyFrom<float>(input1, *gpu_place, context);
+ input2_gpu.CopyFrom<float>(input1, *gpu_place, context);
out_gpu.mutable_data<float>({2, 2}, *gpu_place);
paddle::operators::math::matmul<paddle::platform::GPUPlace, float>(
context, input1_gpu, false, input2_gpu, true, 1, &out_gpu, 0);
- out.CopyFrom<float>(out_gpu, *cpu_place);
+ out.CopyFrom<float>(out_gpu, *cpu_place, context);
float* out_ptr = out.data<float>();
+ context.Wait();
EXPECT_EQ(out_ptr[0], 5);
EXPECT_EQ(out_ptr[1], 14);
EXPECT_EQ(out_ptr[2], 14);
@@ -50,17 +51,18 @@ TEST(math_function, trans_mul_notrans) {
auto* gpu_place = new paddle::platform::GPUPlace(0);
paddle::platform::CUDADeviceContext context(*gpu_place);
- input1_gpu.CopyFrom<float>(input1, *gpu_place);
- input2_gpu.CopyFrom<float>(input1, *gpu_place);
+ input1_gpu.CopyFrom<float>(input1, *gpu_place, context);
+ input2_gpu.CopyFrom<float>(input1, *gpu_place, context);
out_gpu.mutable_data<float>({3, 3}, *gpu_place);
paddle::operators::math::matmul<paddle::platform::GPUPlace, float>(
context, input1_gpu, true, input2_gpu, false, 1, &out_gpu, 0);
- out.CopyFrom<float>(out_gpu, *cpu_place);
+ out.CopyFrom<float>(out_gpu, *cpu_place, context);
float* out_ptr = out.data<float>();
+ context.Wait();
EXPECT_EQ(out_ptr[0], 9);
EXPECT_EQ(out_ptr[1], 12);
EXPECT_EQ(out_ptr[2], 15);
@@ -98,9 +100,9 @@ TEST(math_function, gemm_notrans_cublas) {
auto* gpu_place = new paddle::platform::GPUPlace(0);
paddle::platform::CUDADeviceContext context(*gpu_place);
- input1_gpu.CopyFrom<float>(input1, *gpu_place);
- input2_gpu.CopyFrom<float>(input2, *gpu_place);
- input3_gpu.CopyFrom<float>(input3, *gpu_place);
+ input1_gpu.CopyFrom<float>(input1, *gpu_place, context);
+ input2_gpu.CopyFrom<float>(input2, *gpu_place, context);
+ input3_gpu.CopyFrom<float>(input3, *gpu_place, context);
float* a = input1_gpu.data<float>();
float* b = input2_gpu.data<float>();
float* c = input3_gpu.mutable_data<float>(*gpu_place);
@@ -108,7 +110,7 @@ TEST(math_function, gemm_notrans_cublas) {
paddle::operators::math::gemm<paddle::platform::GPUPlace, float>(
context, false, false, m, n, k, 1, a, 3, b + 1, 4, 1, c + 1, 4);
- input3.CopyFrom<float>(input3_gpu, *cpu_place);
+ input3.CopyFrom<float>(input3_gpu, *cpu_place, context);
// numpy code:
// a = np.arange(6).reshape(2, 3)
@@ -116,6 +118,7 @@ TEST(math_function, gemm_notrans_cublas) {
// c = np.arange(8).reshape(2, 4)[:, 1:]
// out = np.arange(8).reshape(2, 4)
// out[:, 1:] = np.dot(a, b) + c
+ context.Wait();
EXPECT_EQ(input3_ptr[0], 0);
EXPECT_EQ(input3_ptr[1], 24);
EXPECT_EQ(input3_ptr[2], 28);
@@ -152,9 +155,9 @@ TEST(math_function, gemm_trans_cublas) {
auto* gpu_place = new paddle::platform::GPUPlace(0);
paddle::platform::CUDADeviceContext context(*gpu_place);
- input1_gpu.CopyFrom<float>(input1, *gpu_place);
- input2_gpu.CopyFrom<float>(input2, *gpu_place);
- input3_gpu.CopyFrom<float>(input3, *gpu_place);
+ input1_gpu.CopyFrom<float>(input1, *gpu_place, context);
+ input2_gpu.CopyFrom<float>(input2, *gpu_place, context);
+ input3_gpu.CopyFrom<float>(input3, *gpu_place, context);
float* a = input1_gpu.data<float>();
float* b = input2_gpu.data<float>();
float* c = input3_gpu.mutable_data<float>(*gpu_place);
@@ -162,7 +165,8 @@ TEST(math_function, gemm_trans_cublas) {
paddle::operators::math::gemm<paddle::platform::GPUPlace, float>(
context, false, true, m, n, k, 1, a, 3, b + 3, 3, 1, c + 1, 4);
- input3.CopyFrom<float>(input3_gpu, *cpu_place);
+ input3.CopyFrom<float>(input3_gpu, *cpu_place, context);
+ context.Wait();
EXPECT_EQ(input3_ptr[0], 0);
EXPECT_EQ(input3_ptr[1], 24);
diff --git a/paddle/operators/math/vol2col.cc b/paddle/operators/math/vol2col.cc
new file mode 100644
index 0000000000..e9718a0473
--- /dev/null
+++ b/paddle/operators/math/vol2col.cc
@@ -0,0 +1,155 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/operators/math/vol2col.h"
+
+namespace paddle {
+namespace operators {
+namespace math {
+
+/*
+ * vol = [input_channels, input_depth, input_height, input_width]
+ * col =
+ * [input_channels, filter_depth, filter_height, filter_width,
+ * output_depth, output_height, output_width]
+ */
+template <class T>
+class Vol2ColFunctor<platform::CPUPlace, T> {
+ public:
+ void operator()(const platform::DeviceContext& context,
+ const framework::Tensor& vol, framework::Tensor& col,
+ int stride_depth, int stride_height, int stride_width,
+ int padding_depth, int padding_height,
+ int padding_width) const {
+ PADDLE_ENFORCE(vol.dims().size() == 4);
+ PADDLE_ENFORCE(col.dims().size() == 7);
+
+ int input_channels = vol.dims()[0];
+ int input_depth = vol.dims()[1];
+ int input_height = vol.dims()[2];
+ int input_width = vol.dims()[3];
+ int filter_depth = col.dims()[1];
+ int filter_height = col.dims()[2];
+ int filter_width = col.dims()[3];
+ int output_depth = col.dims()[4];
+ int output_height = col.dims()[5];
+ int output_width = col.dims()[6];
+ int channels_col =
+ input_channels * filter_depth * filter_height * filter_width;
+
+ const T* vol_data = vol.data<T>();
+ T* col_data = col.data<T>();
+
+ for (int c = 0; c < channels_col; ++c) {
+ int w_offset = c % filter_width;
+ int h_offset = (c / filter_width) % filter_height;
+ int d_offset = (c / filter_width / filter_height) % filter_depth;
+ int c_in = c / filter_width / filter_height / filter_depth;
+ for (int d = 0; d < output_depth; ++d) {
+ int d_pad = d * stride_depth - padding_depth + d_offset;
+ for (int h = 0; h < output_height; ++h) {
+ int h_pad = h * stride_height - padding_height + h_offset;
+ for (int w = 0; w < output_width; ++w) {
+ int w_pad = w * stride_width - padding_width + w_offset;
+
+ int col_idx =
+ ((c * output_depth + d) * output_height + h) * output_width + w;
+ if (h_pad < 0 || h_pad >= input_height || w_pad < 0 ||
+ w_pad >= input_width || d_pad < 0 || d_pad >= input_depth) {
+ col_data[col_idx] = static_cast<T>(0);
+ } else {
+ int vol_idx =
+ ((c_in * input_depth + d_pad) * input_height + h_pad) *
+ input_width +
+ w_pad;
+ col_data[col_idx] = vol_data[vol_idx];
+ }
+ }
+ }
+ }
+ }
+ }
+};
+
+/*
+ * vol = [input_channels,input_depth, input_height, input_width]
+ * col =
+ * [input_channels, filter_depth, filter_height, filter_width,
+ * output_depth, output_height, output_width]
+ */
+template <class T>
+class Col2VolFunctor<platform::CPUPlace, T> {
+ public:
+ void operator()(const platform::DeviceContext& context,
+ framework::Tensor& vol, const framework::Tensor& col,
+ int stride_depth, int stride_height, int stride_width,
+ int padding_depth, int padding_height,
+ int padding_width) const {
+ PADDLE_ENFORCE(vol.dims().size() == 4);
+ PADDLE_ENFORCE(col.dims().size() == 7);
+
+ int input_channels = vol.dims()[0];
+ int input_depth = vol.dims()[1];
+ int input_height = vol.dims()[2];
+ int input_width = vol.dims()[3];
+ int filter_depth = col.dims()[1];
+ int filter_height = col.dims()[2];
+ int filter_width = col.dims()[3];
+ int output_depth = col.dims()[4];
+ int output_height = col.dims()[5];
+ int output_width = col.dims()[6];
+ int channels_col =
+ input_channels * filter_depth * filter_height * filter_width;
+
+ T* vol_data = vol.data<T>();
+ const T* col_data = col.data<T>();
+
+ for (int c = 0; c < channels_col; ++c) {
+ int w_offset = c % filter_width;
+ int h_offset = (c / filter_width) % filter_height;
+ int d_offset = (c / filter_width / filter_height) % filter_depth;
+ int cIm = c / filter_width / filter_height / filter_depth;
+ for (int d = 0; d < output_depth; ++d) {
+ int d_pad = d * stride_depth - padding_depth + d_offset;
+ for (int h = 0; h < output_height; ++h) {
+ int h_pad = h * stride_height - padding_height + h_offset;
+ for (int w = 0; w < output_width; ++w) {
+ int w_pad = w * stride_width - padding_width + w_offset;
+
+ if (h_pad >= 0 && h_pad < input_height && w_pad >= 0 &&
+ w_pad < input_width && d_pad >= 0 && d_pad < input_depth) {
+ int vol_idx =
+ ((cIm * input_depth + d_pad) * input_height + h_pad) *
+ input_width +
+ w_pad;
+ int col_idx =
+ ((c * output_depth + d) * output_height + h) * output_width +
+ w;
+ vol_data[vol_idx] += col_data[col_idx];
+ }
+ }
+ }
+ }
+ }
+ }
+};
+
+template class Vol2ColFunctor<platform::CPUPlace, float>;
+template class Vol2ColFunctor<platform::CPUPlace, double>;
+template class Col2VolFunctor<platform::CPUPlace, float>;
+template class Col2VolFunctor<platform::CPUPlace, double>;
+
+} // namespace math
+} // namespace operators
+} // namespace paddle
diff --git a/paddle/operators/math/vol2col.cu b/paddle/operators/math/vol2col.cu
new file mode 100644
index 0000000000..27b11fb237
--- /dev/null
+++ b/paddle/operators/math/vol2col.cu
@@ -0,0 +1,204 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/operators/math/vol2col.h"
+#include "paddle/platform/cuda_helper.h"
+
+namespace paddle {
+namespace operators {
+namespace math {
+
+template <class T>
+__global__ void vol2col(int num_kernels, const T* data_vol, int depth,
+ int height, int width, int filter_depth,
+ int filter_height, int filter_width, int stride_depth,
+ int stride_height, int stride_width, int padding_depth,
+ int padding_height, int padding_width, int output_detph,
+ int output_height, int output_width, T* data_col) {
+ for (int index = blockIdx.x * blockDim.x + threadIdx.x; index < num_kernels;
+ index += blockDim.x * gridDim.x) {
+ int w_out = index % output_width;
+ int h_out = (index / output_width) % output_height;
+ int d_out = (index / output_width / output_height) % output_detph;
+ int channel_in = index / output_width / output_height / output_detph;
+ int channel_out = channel_in * filter_depth * filter_height * filter_width;
+ int w_in = w_out * stride_width - padding_width;
+ int h_in = h_out * stride_height - padding_height;
+ int d_in = d_out * stride_depth - padding_depth;
+
+ data_col += ((channel_out * output_detph + d_out) * output_height + h_out) *
+ output_width +
+ w_out;
+ data_vol += ((channel_in * depth + d_in) * height + h_in) * width + w_in;
+ for (int k = 0; k < filter_depth; ++k) {
+ for (int i = 0; i < filter_height; ++i) {
+ for (int j = 0; j < filter_width; ++j) {
+ int d = d_in + k;
+ int h = h_in + i;
+ int w = w_in + j;
+ *data_col = (d >= 0 && d < depth && h >= 0 && h < height && w >= 0 &&
+ w < width)
+ ? data_vol[(k * height + i) * width + j]
+ : 0;
+ data_col += output_detph * output_height * output_width;
+ }
+ }
+ }
+ }
+}
+
+/*
+ * im = [input_channels,intpu_depth, input_height, input_width]
+ * col =
+ * [input_channels, filter_depth, filter_height, filter_width,
+ * output_depth, output_height, output_width]
+ */
+template <class T>
+class Vol2ColFunctor<platform::GPUPlace, T> {
+ public:
+ void operator()(const platform::DeviceContext& context,
+ const framework::Tensor& vol, framework::Tensor& col,
+ int stride_depth, int stride_height, int stride_width,
+ int padding_depth, int padding_height,
+ int padding_width) const {
+ PADDLE_ENFORCE(vol.dims().size() == 4);
+ PADDLE_ENFORCE(col.dims().size() == 7);
+
+ int input_channels = vol.dims()[0];
+ int input_depth = vol.dims()[1];
+ int input_height = vol.dims()[2];
+ int input_width = vol.dims()[3];
+ int filter_depth = col.dims()[1];
+ int filter_height = col.dims()[2];
+ int filter_width = col.dims()[3];
+ int output_depth = col.dims()[4];
+ int output_height = col.dims()[5];
+ int output_width = col.dims()[6];
+
+ int num_outputs =
+ input_channels * output_depth * output_height * output_width;
+
+ const int threads = 1024;
+ const int blocks = (num_outputs + 1024 - 1) / 1024;
+ vol2col<T><<<blocks, threads, 0,
+ reinterpret_cast<const platform::CUDADeviceContext&>(context)
+ .stream()>>>(
+ num_outputs, vol.data<T>(), input_depth, input_height, input_width,
+ filter_depth, filter_height, filter_width, stride_depth, stride_height,
+ stride_width, padding_depth, padding_height, padding_width,
+ output_depth, output_height, output_width, col.data<T>());
+ }
+};
+
+template <class T>
+__global__ void col2vol(int num_kernels, const T* data_col, int depth,
+ int height, int width, int filter_depth,
+ int filter_height, int filter_width, int stride_depth,
+ int stride_height, int stride_width, int padding_depth,
+ int padding_height, int padding_width, int output_detph,
+ int output_height, int output_width, T* data_vol) {
+ for (int index = blockIdx.x * blockDim.x + threadIdx.x; index < num_kernels;
+ index += blockDim.x * gridDim.x) {
+ T src_val = 0;
+ int w = index % width + padding_width;
+ int h = (index / width) % height + padding_height;
+ int d = (index / width / height) % depth + padding_depth;
+ int c = index / width / height / depth;
+ // compute the start and end of the output
+ int w_col_start =
+ (w < filter_width) ? 0 : (w - filter_width) / stride_width + 1;
+ int w_col_end = min(w / stride_width + 1, output_width);
+ int h_col_start =
+ (h < filter_height) ? 0 : (h - filter_height) / stride_height + 1;
+ int h_col_end = min(h / stride_height + 1, output_height);
+ int d_col_start =
+ (d < filter_depth) ? 0 : (d - filter_depth) / stride_depth + 1;
+ int d_col_end = min(d / stride_depth + 1, output_detph);
+
+ int offset = (c * filter_depth * filter_height * filter_width +
+ d * filter_width * filter_height + h * filter_width + w) *
+ output_detph * output_height * output_width;
+
+ int coeff_d_col =
+ (1 - stride_depth * filter_width * filter_height * output_detph) *
+ output_height * output_width;
+ int coeff_h_col =
+ (1 - stride_height * filter_width * output_detph * output_height) *
+ output_width;
+ int coeff_w_col =
+ (1 - stride_width * output_detph * output_height * output_width);
+
+ for (int d_col = d_col_start; d_col < d_col_end; ++d_col) {
+ for (int h_col = h_col_start; h_col < h_col_end; ++h_col) {
+ for (int w_col = w_col_start; w_col < w_col_end; ++w_col) {
+ src_val += data_col[offset + d_col * coeff_d_col +
+ h_col * coeff_h_col + w_col * coeff_w_col];
+ }
+ }
+ }
+ data_vol[index] = src_val;
+ }
+}
+
+/*
+ * im = [input_channels, input_depth, input_height, input_width]
+ * col =
+ * [input_channels, filter_depth, filter_height, filter_width,
+ * output_depth, output_height, output_width]
+ */
+template <class T>
+class Col2VolFunctor<platform::GPUPlace, T> {
+ public:
+ void operator()(const platform::DeviceContext& context,
+ framework::Tensor& vol, const framework::Tensor& col,
+ int stride_depth, int stride_height, int stride_width,
+ int padding_depth, int padding_height,
+ int padding_width) const {
+ PADDLE_ENFORCE(vol.dims().size() == 4);
+ PADDLE_ENFORCE(col.dims().size() == 7);
+
+ int input_channels = vol.dims()[0];
+ int input_depth = vol.dims()[1];
+ int input_height = vol.dims()[2];
+ int input_width = vol.dims()[3];
+ int filter_depth = col.dims()[1];
+ int filter_height = col.dims()[2];
+ int filter_width = col.dims()[3];
+ int output_depth = col.dims()[4];
+ int output_height = col.dims()[5];
+ int output_width = col.dims()[6];
+
+ int num_kernels = input_channels * input_depth * input_height * input_width;
+
+ const int threads = 1024;
+ const int blocks = (num_kernels + 1024 - 1) / 1024;
+
+ col2vol<T><<<blocks, threads, 0,
+ reinterpret_cast<const platform::CUDADeviceContext&>(context)
+ .stream()>>>(
+ num_kernels, col.data<T>(), input_depth, input_height, input_width,
+ filter_depth, filter_height, filter_width, stride_depth, stride_height,
+ stride_width, padding_depth, padding_height, padding_width,
+ output_depth, output_height, output_width, vol.data<T>());
+ }
+};
+
+template class Vol2ColFunctor<platform::GPUPlace, float>;
+template class Vol2ColFunctor<platform::GPUPlace, double>;
+template class Col2VolFunctor<platform::GPUPlace, float>;
+template class Col2VolFunctor<platform::GPUPlace, double>;
+
+} // namespace math
+} // namespace operators
+} // namespace paddle
diff --git a/paddle/operators/math/vol2col.h b/paddle/operators/math/vol2col.h
new file mode 100644
index 0000000000..f022365a16
--- /dev/null
+++ b/paddle/operators/math/vol2col.h
@@ -0,0 +1,78 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+
+#include "paddle/framework/tensor.h"
+#include "paddle/platform/device_context.h"
+
+namespace paddle {
+namespace operators {
+namespace math {
+/*
+ * \brief Converts the feature data of four dimensions(CDHW) into a colData of
+ * seven dimensions in the Vol2ColFunctor calculation,
+ * And in the Col2VolFunctor calculation, it is reversed.
+ *
+ * \param volData Vol data.
+ * \param volShape The shape of volData,
+ * [input_channels, input_depth, input_height, input_width].
+ * \param colData Column data.
+ * \param colShape The shape of colData.
+ *
+ * The shape of colData is:
+ * [input_channels, filter_depth, filter_height, filter_width, output_depth,
+ * output_height, output_width]
+ * So, it is easy to reshape into a convolution matrix for convolution
+ * calculation based on matrix multiplication.
+ * The shape of convolution matrix is [height, width], where the height is equal
+ * input_channels * filter_depth * filter_height * filter_width, and the width
+ * is equal output_depth * output_height * output_width.
+ *
+ * Reshape:
+ * shape of colData shape of convolution matrix
+ * [input_channels,
+ * filter_depth,
+ * filter_height,
+ * filter_width, ======> [height, width]
+ * output_depth,
+ * output_height,
+ * output_width]
+ *
+ * \note The caller needs to ensure that volShape.inputChannels is equal to
+ * colShape.inputChannels.
+ */
+template <typename Place, typename T>
+class Vol2ColFunctor {
+ public:
+ void operator()(const platform::DeviceContext& context,
+ const framework::Tensor& vol, framework::Tensor& col,
+ int stride_depth, int stride_height, int stride_width,
+ int padding_depth, int padding_height,
+ int padding_width) const;
+};
+
+template <typename Place, typename T>
+class Col2VolFunctor {
+ public:
+ void operator()(const platform::DeviceContext& context,
+ framework::Tensor& vol, const framework::Tensor& col,
+ int stride_depth, int stride_height, int stride_width,
+ int padding_depth, int padding_height,
+ int padding_width) const;
+};
+
+} // namespace math
+} // namespace operators
+} // namespace paddle
diff --git a/paddle/operators/math/vol2col_test.cc b/paddle/operators/math/vol2col_test.cc
new file mode 100644
index 0000000000..2d69218843
--- /dev/null
+++ b/paddle/operators/math/vol2col_test.cc
@@ -0,0 +1,135 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/operators/math/vol2col.h"
+#include <gtest/gtest.h>
+#include <iostream>
+
+template <typename Place>
+void testVol2col() {
+ paddle::framework::Tensor input;
+ paddle::framework::Tensor input_tmp;
+ paddle::framework::Tensor output;
+ paddle::framework::Tensor output_tmp;
+
+ auto* place = new Place();
+ paddle::platform::DeviceContext* context;
+ if (paddle::platform::is_cpu_place(*place)) {
+ context =
+ new paddle::platform::CPUDeviceContext(paddle::platform::CPUPlace());
+ } else {
+#ifdef PADDLE_WITH_CUDA
+ context =
+ new paddle::platform::CUDADeviceContext(paddle::platform::GPUPlace());
+#else
+ PADDLE_THROW("no GPU support");
+#endif // PADDLE_WITH_CUDA
+ }
+
+ /**
+ * input = [[0, 1, 2,
+ * 3, 4, 5]
+ * [6, 7, 8,
+ * 9, 10, 11]]
+ *
+ * output = [0, 1
+ * 1, 2
+ * 3, 4
+ * 4, 5
+ * 6, 7
+ * 7, 8
+ * 9, 10
+ * 10, 11]
+ *
+ * col2vol = [[0, 2, 2,
+ * 3, 8, 5]
+ * [6, 14, 8,
+ * 9, 20, 11]]
+ *
+ */
+ int input_depth = 2;
+ int input_height = 2;
+ int input_width = 3;
+ int filter_size = 2;
+ int stride = 1;
+ int padding = 0;
+ int output_depth = (input_depth - filter_size + 2 * padding) / stride + 1;
+ int output_height = (input_height - filter_size + 2 * padding) / stride + 1;
+ int output_width = (input_width - filter_size + 2 * padding) / stride + 1;
+
+ // Vol2Col test
+ float* input_ptr =
+ input_tmp.mutable_data<float>({1, input_depth, input_height, input_width},
+ paddle::platform::CPUPlace());
+ float arr[12] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11};
+ memcpy(input_ptr, arr, 12 * sizeof(float));
+
+ if (paddle::platform::is_cpu_place(*place)) {
+ input = input_tmp;
+ } else {
+ input.CopyFrom<float>(input_tmp, *place, *context);
+ }
+ output.mutable_data<float>({1, filter_size, filter_size, filter_size,
+ output_depth, output_height, output_width},
+ *place);
+
+ paddle::operators::math::Vol2ColFunctor<Place, float> vol2col;
+ vol2col(*context, input, output, stride, stride, stride, padding, padding,
+ padding);
+
+ float vol_2_col[] = {0, 1, 1, 2, 3, 4, 4, 5, 6, 7, 7, 8, 9, 10, 10, 11};
+ float* out_cfo_ptr;
+ if (paddle::platform::is_cpu_place(*place)) {
+ out_cfo_ptr = output.data<float>();
+ } else {
+ output_tmp.CopyFrom<float>(output, paddle::platform::CPUPlace(), *context);
+ out_cfo_ptr = output_tmp.data<float>();
+ }
+
+ for (int i = 0; i < 16; ++i) {
+ EXPECT_EQ(out_cfo_ptr[i], vol_2_col[i]);
+ }
+
+ // Col2Vol test
+ float col_2_vol[] = {0, 2, 2, 3, 8, 5, 6, 14, 8, 9, 20, 11};
+ memset(input_ptr, 0, 12 * sizeof(float));
+ if (paddle::platform::is_cpu_place(*place)) {
+ input = input_tmp;
+ } else {
+ input.CopyFrom<float>(input_tmp, *place, *context);
+ }
+
+ paddle::operators::math::Col2VolFunctor<Place, float> col2vol;
+ col2vol(*context, input, output, stride, stride, stride, padding, padding,
+ padding);
+
+ float* in_ptr;
+ if (paddle::platform::is_cpu_place(*place)) {
+ in_ptr = input.data<float>();
+ } else {
+ input_tmp.CopyFrom<float>(input, paddle::platform::CPUPlace(), *context);
+ in_ptr = input_tmp.data<float>();
+ }
+
+ for (int i = 0; i < 12; ++i) {
+ EXPECT_EQ(in_ptr[i], col_2_vol[i]);
+ }
+}
+
+TEST(math, vol2col) {
+ testVol2col<paddle::platform::CPUPlace>();
+#ifdef PADDLE_WITH_CUDA
+ testVol2col<paddle::platform::GPUPlace>();
+#endif // PADDLE_WITH_CUDA
+}
diff --git a/paddle/operators/multiplex_op.cc b/paddle/operators/multiplex_op.cc
index a86685b6dd..051051b051 100644
--- a/paddle/operators/multiplex_op.cc
+++ b/paddle/operators/multiplex_op.cc
@@ -115,8 +115,9 @@ class MultiplexGradOp : public framework::OperatorWithKernel {
} // namespace paddle
namespace ops = paddle::operators;
-REGISTER_OP(multiplex, ops::MultiplexOp, ops::MultiplexOpMaker, multiplex_grad,
- ops::MultiplexGradOp);
+REGISTER_OPERATOR(multiplex, ops::MultiplexOp, ops::MultiplexOpMaker,
+ paddle::framework::DefaultGradOpDescMaker<false>);
+REGISTER_OPERATOR(multiplex_grad, ops::MultiplexGradOp);
REGISTER_OP_CPU_KERNEL(
multiplex, ops::MultiplexCPUKernel<paddle::platform::CPUPlace, float>);
REGISTER_OP_CPU_KERNEL(
diff --git a/paddle/operators/multiplex_op.cu b/paddle/operators/multiplex_op.cu
index 72b1f96eaf..10cb0e005f 100644
--- a/paddle/operators/multiplex_op.cu
+++ b/paddle/operators/multiplex_op.cu
@@ -33,7 +33,8 @@ class MultiplexGPUKernel : public framework::OpKernel<T> {
auto cols = ins[0]->numel() / rows;
// copy index to cpu
Tensor index_t_cpu;
- index_t_cpu.CopyFrom<int32_t>(*ids, platform::CPUPlace());
+ index_t_cpu.CopyFrom<int32_t>(*ids, platform::CPUPlace(),
+ ctx.device_context());
auto* index = index_t_cpu.data<int32_t>();
auto stream = reinterpret_cast<const platform::CUDADeviceContext&>(
ctx.device_context())
@@ -70,7 +71,8 @@ class MultiplexGradGPUKernel : public framework::OpKernel<T> {
auto cols = ins[0]->numel() / rows;
// copy index to cpu
Tensor index_t_cpu;
- index_t_cpu.CopyFrom<int32_t>(*ids, platform::CPUPlace());
+ index_t_cpu.CopyFrom<int32_t>(*ids, platform::CPUPlace(),
+ ctx.device_context());
auto* index = index_t_cpu.data<int32_t>();
auto stream = reinterpret_cast<const platform::CUDADeviceContext&>(
diff --git a/paddle/operators/pool_op.cc b/paddle/operators/pool_op.cc
index ba3b5ed207..c6d9aae133 100644
--- a/paddle/operators/pool_op.cc
+++ b/paddle/operators/pool_op.cc
@@ -22,157 +22,181 @@ int OutputSizePool(int input_size, int filter_size, int padding, int stride) {
return output_size;
}
-class PoolOp : public framework::OperatorWithKernel {
- public:
- using framework::OperatorWithKernel::OperatorWithKernel;
-
- protected:
- void InferShape(framework::InferShapeContext *ctx) const override {
- PADDLE_ENFORCE(ctx->HasInput("X"),
- "X(Input) of Pooling should not be null.");
- PADDLE_ENFORCE(ctx->HasOutput("Out"),
- "Out(Output) of Pooling should not be null.");
-
- auto in_x_dims = ctx->GetInputDim("X");
-
- std::string pooling_type = ctx->Attrs().Get<std::string>("poolingType");
- std::vector<int> ksize = ctx->Attrs().Get<std::vector<int>>("ksize");
- std::vector<int> strides = ctx->Attrs().Get<std::vector<int>>("strides");
- std::vector<int> paddings = ctx->Attrs().Get<std::vector<int>>("paddings");
-
- PADDLE_ENFORCE(pooling_type == "max" || pooling_type == "avg",
- "pooling_type should be 'max' or 'avg'");
- PADDLE_ENFORCE(in_x_dims.size() == 4 || in_x_dims.size() == 5,
- "Pooling intput should be 4-D or 5-D");
-
- if (ctx->Attrs().Get<bool>("globalPooling")) {
- ksize.resize(static_cast<size_t>(in_x_dims.size()) - 2);
- for (size_t i = 0; i < ksize.size(); ++i)
- ksize[i] = static_cast<int>(in_x_dims[i + 2]);
- }
-
- PADDLE_ENFORCE(in_x_dims.size() - ksize.size() == 2U,
- "Input size and Pooling size should be consistent.");
- PADDLE_ENFORCE(ksize.size() == 2 || ksize.size() == 3,
- "Pooling size should be 2 elements. or 3 elements.");
- PADDLE_ENFORCE_EQ(ksize.size(), strides.size(),
- "strides size and pooling size should be the same.");
- PADDLE_ENFORCE_EQ(ksize.size(), paddings.size(),
- "paddings size and pooling size should be the same.");
-
- std::vector<int64_t> output_shape({in_x_dims[0], in_x_dims[1]});
- for (size_t i = 0; i < ksize.size(); ++i) {
- output_shape.push_back(
- OutputSizePool(in_x_dims[i + 2], ksize[i], paddings[i], strides[i]));
- }
- ctx->SetOutputDim("Out", framework::make_ddim(output_shape));
+void PoolOp::InferShape(framework::InferShapeContext *ctx) const {
+ PADDLE_ENFORCE(ctx->HasInput("X"), "X(Input) of Pooling should not be null.");
+ PADDLE_ENFORCE(ctx->HasOutput("Out"),
+ "Out(Output) of Pooling should not be null.");
+
+ auto in_x_dims = ctx->GetInputDim("X");
+
+ std::string pooling_type = ctx->Attrs().Get<std::string>("poolingType");
+ std::vector<int> ksize = ctx->Attrs().Get<std::vector<int>>("ksize");
+ std::vector<int> strides = ctx->Attrs().Get<std::vector<int>>("strides");
+ std::vector<int> paddings = ctx->Attrs().Get<std::vector<int>>("paddings");
+
+ PADDLE_ENFORCE(in_x_dims.size() == 4 || in_x_dims.size() == 5,
+ "Pooling intput should be 4-D or 5-D tensor.");
+
+ if (ctx->Attrs().Get<bool>("globalPooling")) {
+ ksize.resize(static_cast<size_t>(in_x_dims.size()) - 2);
+ for (size_t i = 0; i < ksize.size(); ++i)
+ ksize[i] = static_cast<int>(in_x_dims[i + 2]);
}
-};
-
-class PoolOpGrad : public framework::OperatorWithKernel {
- public:
- using framework::OperatorWithKernel::OperatorWithKernel;
-
- protected:
- void InferShape(framework::InferShapeContext *ctx) const override {
- PADDLE_ENFORCE(ctx->HasInput("X"),
- "X(Input) of Pooling should not be null.");
- PADDLE_ENFORCE(ctx->HasOutput(framework::GradVarName("X")),
- "Input@Grad of Pooling should not be null.");
- ctx->SetOutputDim(framework::GradVarName("X"), ctx->GetInputDim("X"));
+
+ PADDLE_ENFORCE(in_x_dims.size() - ksize.size() == 2U,
+ "Input size and pooling size should be consistent.");
+ PADDLE_ENFORCE_EQ(ksize.size(), strides.size(),
+ "Strides size and pooling size should be the same.");
+ PADDLE_ENFORCE_EQ(ksize.size(), paddings.size(),
+ "Paddings size and pooling size should be the same.");
+
+ std::vector<int64_t> output_shape({in_x_dims[0], in_x_dims[1]});
+ for (size_t i = 0; i < ksize.size(); ++i) {
+ output_shape.push_back(
+ OutputSizePool(in_x_dims[i + 2], ksize[i], paddings[i], strides[i]));
}
-};
-
-class Pool2dOpMaker : public framework::OpProtoAndCheckerMaker {
- public:
- Pool2dOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker)
- : OpProtoAndCheckerMaker(proto, op_checker) {
- AddInput(
- "X",
- "The input tensor of pooling operator. "
- "The format of input tensor is NCHW. Where N is batch size, C is the "
- "number of channels, H and W is the height and width of feature.");
- AddOutput("Out",
- "The output tensor of pooling operator."
- "The format of output tensor is also NCHW.");
-
- AddAttr<std::string>("poolingType",
- "PoolingType of pooling operator."
- "Str constant equal to 'max' or 'avg'.")
- .InEnum({"max", "avg"});
- AddAttr<std::vector<int>>(
- "ksize",
- "Pooling size(depth, height, width) of pooling operator."
- "If globalPooling = true, ksize is ignored and need not be "
- "specified."); // TODO(Add checker)
- AddAttr<bool>(
- "globalPooling",
- "Whether to use the globalPooling."
- "Bool constant equal to false or true."
- "Default false."
- "If globalPooling = true, ksize is ignored and need not be specified.")
- .SetDefault(false);
- AddAttr<std::vector<int>>("strides",
- "Strides(height, width) of pooling operator."
- "Default {1,1}")
- .SetDefault({1, 1}); // TODO(Add checker)
- AddAttr<std::vector<int>>("paddings",
- "Paddings(height, width) of pooling operator."
- "Default {0,0}.")
- .SetDefault({0, 0}); // TODO(Add checker)
- AddComment(R"DOC(
+ ctx->SetOutputDim("Out", framework::make_ddim(output_shape));
+}
+
+void PoolOpGrad::InferShape(framework::InferShapeContext *ctx) const {
+ PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) must not be null.");
+ PADDLE_ENFORCE(ctx->HasOutput(framework::GradVarName("X")),
+ "Input(X@GRAD) should not be null.");
+ ctx->SetOutputDim(framework::GradVarName("X"), ctx->GetInputDim("X"));
+}
+
+Pool2dOpMaker::Pool2dOpMaker(framework::OpProto *proto,
+ framework::OpAttrChecker *op_checker)
+ : OpProtoAndCheckerMaker(proto, op_checker) {
+ AddInput(
+ "X",
+ "(Tensor) The input tensor of pooling operator. "
+ "The format of input tensor is NCHW. Where N is batch size, C is the "
+ "number of channels, H and W is the height and width of feature.");
+ AddOutput("Out",
+ "(Tensor) The output tensor of pooling operator."
+ "The format of output tensor is also NCHW."
+ "Where N is batch size, C is "
+ "the number of channels, H and W is the height and "
+ "width of feature.");
+
+ AddAttr<std::string>("poolingType",
+ "PoolingType of pooling operator."
+ "Str constant equal to 'max' or 'avg'.")
+ .InEnum({"max", "avg"});
+
+ AddAttr<std::vector<int>>(
+ "ksize",
+ "The pooling window size(height, width) of pooling operator."
+ "If globalPooling = true, ksize is ignored and need not be "
+ "specified."); // TODO(Chengduo): Add checker. (Currently,
+ // TypedAttrChecker don't support vector type.)
+ AddAttr<bool>(
+ "globalPooling",
+ "Whether to use the globalPooling."
+ "Bool constant equal to false or true."
+ "Default false."
+ "If globalPooling = true, ksize is ignored and need not be specified.")
+ .SetDefault(false);
+ AddAttr<std::vector<int>>("strides",
+ "The strides(height, width) of pooling window."
+ "Default {1,1}.")
+ .SetDefault({1, 1}); // TODO(Chengduo): Add checker. (Currently,
+ // TypedAttrChecker don't support vector type.)
+ AddAttr<std::vector<int>>("paddings",
+ "The zero padding(height, width) size on both sides"
+ "Default {0,0}.")
+ .SetDefault({0, 0}); // TODO(Chengduo): Add checker. (Currently,
+ // TypedAttrChecker don't support vector type.)
+
+ AddComment(R"DOC(
The pooling2d operation calculates the output based on
the input, poolingType and ksize, strides, paddings parameters.
+Input(X) and output(Out) are in NCHW format. Where N is batch size, C is the
+number of channels, H and W is the height and width of feature.
+Parameters(ksize, strides, paddings) are two elements.
+These two elements represent height and width, respectively.
+The input(X) size and output(Out) size may be different.
+
+Example:
+ Input:
+ X shape: (N, C, H_in, W_in)
+ Output:
+ Out shape: (N, C, H_out, W_out)
+ Mask shape: (N, C, H_out, W_out)
+ where
+ H_out = (H_in - ksize[0] + 2 * paddings[0]) / strides[0] + 1;
+ W_out = (W_in - ksize[1] + 2 * paddings[1]) / strides[1] + 1;
)DOC");
- }
-};
-
-class Pool3dOpMaker : public framework::OpProtoAndCheckerMaker {
- public:
- Pool3dOpMaker(framework::OpProto *proto, framework::OpAttrChecker *op_checker)
- : OpProtoAndCheckerMaker(proto, op_checker) {
- AddInput("X",
- "The input tensor of pooling operator. "
- "The format of input tensor is NCDHW. Where N is batch size, C is "
- "the "
- "number of channels, D, H and W is the depth, height and width of "
- "feature.");
- AddOutput("Out",
- "The output tensor of pooling operator."
- "The format of output tensor is also NCDHW.");
-
- AddAttr<std::string>("poolingType",
- "PoolingType of pooling operator."
- "str constant equal to 'max' or 'avg'.")
- .InEnum({"max", "avg"});
- AddAttr<std::vector<int>>(
- "ksize",
- "Pooling size(depth, height, width) of pooling operator."
- "If globalPooling = true, ksize is ignored and need not be "
- "specified."); // TODO(Add checker)
- AddAttr<bool>(
- "globalPooling",
- "Whether to use the globalPooling."
- "Bool constant equal to false or true."
- "Default false."
- "If globalPooling = true, ksize is ignored and need not be specified.")
- .SetDefault(false);
- AddAttr<std::vector<int>>(
- "strides",
- "Strides(depth, height, width) of pooling operator."
- "Default {1,1,1}.")
- .SetDefault({1, 1, 1}); // TODO(Add checker)
- AddAttr<std::vector<int>>(
- "paddings",
- "Paddings(depth, height, width) of pooling operator."
- "Default {0,0,0}.")
- .SetDefault({0, 0, 0}); // TODO(Add checker)
- AddComment(R"DOC(
+}
+
+Pool3dOpMaker::Pool3dOpMaker(framework::OpProto *proto,
+ framework::OpAttrChecker *op_checker)
+ : OpProtoAndCheckerMaker(proto, op_checker) {
+ AddInput(
+ "X",
+ "(Tensor) The input tensor of pooling operator. "
+ "The format of input tensor is NCDHW. Where N is batch size, C is "
+ "the number of channels, D, H and W is the depth, height and width of "
+ "feature.");
+ AddOutput("Out",
+ "(Tensor) The output tensor of pooling operator."
+ "The format of output tensor is also NCDHW."
+ "Where N is batch size, C is "
+ "the number of channels, D, H and W is the depth, height and "
+ "width of feature.");
+
+ AddAttr<std::string>("poolingType",
+ "PoolingType of pooling operator."
+ "Str constant equal to 'max' or 'avg'.")
+ .InEnum({"max", "avg"});
+
+ AddAttr<std::vector<int>>(
+ "ksize",
+ "The pooling window size(depth, height, width) of pooling operator."
+ "If globalPooling = true, ksize is ignored and need not be "
+ "specified."); // TODO(Chengduo): Add checker. (Currently,
+ // TypedAttrChecker don't support vector type.)
+ AddAttr<bool>(
+ "globalPooling",
+ "Whether to use the globalPooling."
+ "Bool constant equal to false or true."
+ "Default false."
+ "If globalPooling = true, ksize is ignored and need not be specified.")
+ .SetDefault(false);
+ AddAttr<std::vector<int>>("strides",
+ "Strides(depth, height, width) of pooling operator."
+ "Default {1,1,1}.")
+ .SetDefault({1, 1, 1}); // TODO(Chengduo): Add checker. (Currently,
+ // TypedAttrChecker don't support vector type.)
+ AddAttr<std::vector<int>>(
+ "paddings",
+ "Paddings(depth, height, width) of pooling operator."
+ "Default {0,0,0}.")
+ .SetDefault({0, 0, 0}); // TODO(Chengduo): Add checker. (Currently,
+ // TypedAttrChecker don't support vector type.)
+
+ AddComment(R"DOC(
The pooling3d operation calculates the output based on
the input, poolingType and ksize, strides, paddings parameters.
+Input(X) and output(Out) are in NCDHW format. Where N is batch
+size, C is the number of channels, D, H and W is the depth, height and
+width of feature. Parameters(ksize, strides, paddings) are three elements.
+These three elements represent depth, height and width, respectively.
+The input(X) size and output(Out) size may be different.
+
+Example:
+ Input:
+ X shape: (N, C, D_in, H_in, W_in)
+ Output:
+ Out shape: (N, C, D_out, H_out, W_out)
+ Mask shape: (N, C, D_out, H_out, W_out)
+ where
+ D_out = (D_in - ksize[0] + 2 * paddings[0]) / strides[0] + 1;
+ H_out = (H_in - ksize[1] + 2 * paddings[1]) / strides[1] + 1;
+ W_out = (W_in - ksize[2] + 2 * paddings[2]) / strides[2] + 1;
)DOC");
- }
-};
+}
} // namespace operators
} // namespace paddle
diff --git a/paddle/operators/pool_op.h b/paddle/operators/pool_op.h
index c2bc358def..e5016d573d 100644
--- a/paddle/operators/pool_op.h
+++ b/paddle/operators/pool_op.h
@@ -24,6 +24,34 @@ namespace operators {
using Tensor = framework::Tensor;
+class PoolOp : public framework::OperatorWithKernel {
+ public:
+ using framework::OperatorWithKernel::OperatorWithKernel;
+
+ protected:
+ void InferShape(framework::InferShapeContext* ctx) const override;
+};
+
+class PoolOpGrad : public framework::OperatorWithKernel {
+ public:
+ using framework::OperatorWithKernel::OperatorWithKernel;
+
+ protected:
+ void InferShape(framework::InferShapeContext* ctx) const override;
+};
+
+class Pool2dOpMaker : public framework::OpProtoAndCheckerMaker {
+ public:
+ Pool2dOpMaker(framework::OpProto* proto,
+ framework::OpAttrChecker* op_checker);
+};
+
+class Pool3dOpMaker : public framework::OpProtoAndCheckerMaker {
+ public:
+ Pool3dOpMaker(framework::OpProto* proto,
+ framework::OpAttrChecker* op_checker);
+};
+
template <typename Place, typename T>
class PoolKernel : public framework::OpKernel<T> {
public:
diff --git a/paddle/operators/pool_with_index_op.cc b/paddle/operators/pool_with_index_op.cc
index 7b6afcfd1f..005ee88693 100644
--- a/paddle/operators/pool_with_index_op.cc
+++ b/paddle/operators/pool_with_index_op.cc
@@ -43,7 +43,7 @@ class MaxPoolWithIndexOp : public framework::OperatorWithKernel {
std::vector<int> paddings = ctx->Attrs().Get<std::vector<int>>("paddings");
PADDLE_ENFORCE(in_x_dims.size() == 4 || in_x_dims.size() == 5,
- "Pooling intput should be 4-D or 5-D");
+ "Pooling intput should be 4-D or 5-D tensor.");
if (ctx->Attrs().Get<bool>("globalPooling")) {
ksize.resize(static_cast<size_t>(in_x_dims.size()) - 2);
@@ -52,7 +52,7 @@ class MaxPoolWithIndexOp : public framework::OperatorWithKernel {
}
PADDLE_ENFORCE(in_x_dims.size() - ksize.size() == 2U,
- "Intput size and pooling size should be consistent.");
+ "Input size and pooling size should be consistent.");
PADDLE_ENFORCE_EQ(ksize.size(), strides.size(),
"Strides size and pooling size should be the same.");
PADDLE_ENFORCE_EQ(ksize.size(), paddings.size(),
@@ -74,6 +74,7 @@ class MaxPoolWithIndexOpGrad : public framework::OperatorWithKernel {
protected:
void InferShape(framework::InferShapeContext *ctx) const override {
+ PADDLE_ENFORCE(ctx->HasInput("Mask"), "Input(Mask) must not be null.");
PADDLE_ENFORCE(ctx->HasInput("X"), "Input(X) must not be null.");
PADDLE_ENFORCE(ctx->HasOutput(framework::GradVarName("X")),
"Input(X@GRAD) should not be null.");
@@ -88,17 +89,17 @@ class MaxPool2dWithIndexOpMaker : public framework::OpProtoAndCheckerMaker {
: OpProtoAndCheckerMaker(proto, op_checker) {
AddInput(
"X",
- "The input tensor of pooling operator. "
+ "(Tensor) The input tensor of pooling operator. "
"The format of input tensor is NCHW. Where N is batch size, C is the "
"number of channels, H and W is the height and width of image.");
AddOutput("Out",
- "The output tensor of pooling operator."
+ "(Tensor) The output tensor of pooling operator."
"The format of output tensor is also NCHW."
"Where N is batch size, C is "
"the number of channels, H and W is the height and "
"width of image.");
AddOutput("Mask",
- "The Mask tensor of pooling operator."
+ "(Tensor) The Mask tensor of pooling operator."
"The format of output tensor is also NCHW."
"Where N is batch size, C is the number of channels, H and W "
"is the height and width of image."
@@ -106,7 +107,7 @@ class MaxPool2dWithIndexOpMaker : public framework::OpProtoAndCheckerMaker {
AddAttr<std::vector<int>>(
"ksize",
- "The pooling size(height, width) of pooling operator."
+ "The pooling window size(height, width) of pooling operator."
"If globalPooling = true, ksize is ignored and need not be "
"specified."); // TODO(Chengduo): Add checker. (Currently,
// TypedAttrChecker don't support vector type.)
@@ -118,13 +119,14 @@ class MaxPool2dWithIndexOpMaker : public framework::OpProtoAndCheckerMaker {
"If globalPooling = true, ksize is ignored and need not be specified.")
.SetDefault(false);
AddAttr<std::vector<int>>("strides",
- "Strides(height, width) of pooling operator."
+ "The strides(height, width) of pooling window."
"Default {1,1}.")
.SetDefault({1, 1}); // TODO(Chengduo): Add checker. (Currently,
// TypedAttrChecker don't support vector type.)
- AddAttr<std::vector<int>>("paddings",
- "Paddings(height, width) of pooling operator."
- "Default {0,0}.")
+ AddAttr<std::vector<int>>(
+ "paddings",
+ "The zero padding(height, width) size on both sides"
+ "Default {0,0}.")
.SetDefault({0, 0}); // TODO(Chengduo): Add checker. (Currently,
// TypedAttrChecker don't support vector type.)
@@ -135,6 +137,17 @@ output(Out, Mask) are in NCHW format. Where N is batch size, C is the
number of channels, H and W is the height and width of feature.
Parameters(ksize, strides, paddings) are two elements.
These two elements represent height and width, respectively.
+The input(X) size and output(Out, Mask) size may be different.
+
+Example:
+ Input:
+ X shape: (N, C, H_in, W_in)
+ Output:
+ Out shape: (N, C, H_out, W_out)
+ Mask shape: (N, C, H_out, W_out)
+ where
+ H_out = (H_in - ksize[0] + 2 * paddings[0]) / strides[0] + 1;
+ W_out = (W_in - ksize[1] + 2 * paddings[1]) / strides[1] + 1;
)DOC");
}
};
@@ -146,18 +159,18 @@ class MaxPool3dWithIndexOpMaker : public framework::OpProtoAndCheckerMaker {
: OpProtoAndCheckerMaker(proto, op_checker) {
AddInput(
"X",
- "The input tensor of pooling operator. "
+ "(Tensor) The input tensor of pooling operator. "
"The format of input tensor is NCDHW. Where N is batch size, C is "
"the number of channels, D, H and W is the depth, height and width of "
"image.");
AddOutput("Out",
- "The output tensor of pooling operator."
+ "(Tensor) The output tensor of pooling operator."
"The format of output tensor is also NCDHW."
"Where N is batch size, C is "
"the number of channels, D, H and W is the depth, height and "
"width of image.");
AddOutput("Mask",
- "The Mask tensor of pooling operator."
+ "(Tensor) The Mask tensor of pooling operator."
"The format of output tensor is also NCDHW."
"Where N is batch size, C is the number of channels, D, H and W "
"is the depth, height and width of image."
@@ -165,7 +178,7 @@ class MaxPool3dWithIndexOpMaker : public framework::OpProtoAndCheckerMaker {
AddAttr<std::vector<int>>(
"ksize",
- "The pooling size(depth, height, width) of pooling operator."
+ "The pooling window size(depth, height, width) of pooling operator."
"If globalPooling = true, ksize is ignored and need not be "
"specified."); // TODO(Chengduo): Add checker. (Currently,
// TypedAttrChecker don't support vector type.)
@@ -196,6 +209,18 @@ Input(X) and output(Out, Mask) are in NCDHW format. Where N is batch
size, C is the number of channels, D, H and W is the depth, height and
width of feature. Parameters(ksize, strides, paddings) are three elements.
These three elements represent depth, height and width, respectively.
+The input(X) size and output(Out, Mask) size may be different.
+
+Example:
+ Input:
+ X shape: (N, C, D_in, H_in, W_in)
+ Output:
+ Out shape: (N, C, D_out, H_out, W_out)
+ Mask shape: (N, C, D_out, H_out, W_out)
+ where
+ D_out = (D_in - ksize[0] + 2 * paddings[0]) / strides[0] + 1;
+ H_out = (H_in - ksize[1] + 2 * paddings[1]) / strides[1] + 1;
+ W_out = (W_in - ksize[2] + 2 * paddings[2]) / strides[2] + 1;
)DOC");
}
};
diff --git a/paddle/operators/recurrent_op.cc b/paddle/operators/recurrent_op.cc
index 04c4c24951..00647f55f7 100644
--- a/paddle/operators/recurrent_op.cc
+++ b/paddle/operators/recurrent_op.cc
@@ -46,7 +46,7 @@ void RecurrentAlgorithm::Run(const Scope& scope,
}
(*stepnet_)->Run(*step_scopes[step_id], dev_ctx);
}
- rnn::ConcatOutputs(step_scopes, arg_->outlinks, seq_len);
+ rnn::ConcatOutputs(step_scopes, arg_->outlinks, seq_len, dev_ctx);
}
void RecurrentAlgorithm::CreateScopes(const Scope& scope,
@@ -151,12 +151,12 @@ void RecurrentGradientAlgorithm::Run(
auto& step_scopes = GetStepScopes(scope);
rnn::SegmentInputs(step_scopes, arg_->inlinks, seq_len);
for (int step_id = seq_len - 1; step_id >= 0; --step_id) {
- if (step_id != seq_len - 1) {
+ if (static_cast<size_t>(step_id) != seq_len - 1) {
rnn::LinkMemories(step_scopes, arg_->memories, step_id, 1);
}
(*stepnet_)->Run(*step_scopes[step_id], dev_ctx);
}
- rnn::ConcatOutputs(step_scopes, arg_->outlinks, seq_len);
+ rnn::ConcatOutputs(step_scopes, arg_->outlinks, seq_len, dev_ctx);
LinkBootMemoryGradients(step_scopes[0]);
}
diff --git a/paddle/operators/reshape_op.h b/paddle/operators/reshape_op.h
index 628dfe4c0f..3ba4611458 100644
--- a/paddle/operators/reshape_op.h
+++ b/paddle/operators/reshape_op.h
@@ -33,7 +33,7 @@ class ReshapeKernel : public framework::OpKernel<T> {
std::transform(shape.begin(), shape.end(), shape_int64.begin(),
[](int a) { return static_cast<int64_t>(a); });
auto out_dims = framework::make_ddim(shape_int64);
- out->CopyFrom<T>(*in, ctx.GetPlace());
+ out->CopyFrom<T>(*in, ctx.GetPlace(), ctx.device_context());
out->Resize(out_dims);
}
};
@@ -47,7 +47,7 @@ class ReshapeGradKernel : public framework::OpKernel<T> {
d_x->mutable_data<T>(ctx.GetPlace());
auto in_dims = d_x->dims();
- d_x->CopyFrom<T>(*d_out, ctx.GetPlace());
+ d_x->CopyFrom<T>(*d_out, ctx.GetPlace(), ctx.device_context());
d_x->Resize(in_dims);
}
};
diff --git a/paddle/operators/rnn/recurrent_op_utils.cc b/paddle/operators/rnn/recurrent_op_utils.cc
index ef317a71f1..d264664a99 100644
--- a/paddle/operators/rnn/recurrent_op_utils.cc
+++ b/paddle/operators/rnn/recurrent_op_utils.cc
@@ -51,7 +51,7 @@ void SegmentInputs(const std::vector<Scope*>& step_scopes,
void ConcatOutputs(const std::vector<Scope*>& step_scopes,
const std::vector<std::string>& outlinks,
- const size_t seq_len) {
+ const size_t seq_len, const platform::DeviceContext& ctx) {
for (size_t i = 0; i < outlinks.size(); i++) {
auto* output_var = step_scopes[0]->parent().FindVar(outlinks[i]);
PADDLE_ENFORCE_NOT_NULL(output_var, "output link [%s] is not in scope.",
@@ -72,7 +72,7 @@ void ConcatOutputs(const std::vector<Scope*>& step_scopes,
// TODO(luotao02) data type and platform::DeviceContext() should set
// correctly
(output->Slice<float>(j, j + 1))
- .CopyFrom<float>(*step_output, platform::CPUPlace());
+ .CopyFrom<float>(*step_output, platform::CPUPlace(), ctx);
}
}
}
diff --git a/paddle/operators/rnn/recurrent_op_utils.h b/paddle/operators/rnn/recurrent_op_utils.h
index fd17b9b889..fe173edb24 100644
--- a/paddle/operators/rnn/recurrent_op_utils.h
+++ b/paddle/operators/rnn/recurrent_op_utils.h
@@ -71,7 +71,7 @@ void SegmentInputs(const std::vector<Scope*>& step_scopes,
*/
void ConcatOutputs(const std::vector<Scope*>& step_scopes,
const std::vector<std::string>& outlinks,
- const size_t seq_len);
+ const size_t seq_len, const platform::DeviceContext& ctx);
void LinkMemories(const std::vector<Scope*>& step_scopes,
const std::vector<MemoryAttr>& memories, const size_t step_id,
diff --git a/paddle/operators/sequence_concat_op.cc b/paddle/operators/sequence_concat_op.cc
new file mode 100644
index 0000000000..287fb1942e
--- /dev/null
+++ b/paddle/operators/sequence_concat_op.cc
@@ -0,0 +1,129 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/operators/sequence_concat_op.h"
+
+namespace paddle {
+namespace operators {
+
+class SequenceConcatOp : public framework::OperatorWithKernel {
+ public:
+ using framework::OperatorWithKernel::OperatorWithKernel;
+
+ protected:
+ void InferShape(framework::InferShapeContext* ctx) const override {
+ PADDLE_ENFORCE(ctx->HasInputs("X"),
+ "Inputs(X) of SequenceConcatOp should not be null.");
+ PADDLE_ENFORCE(ctx->HasOutput("Out"),
+ "Output(Out) of SequenceConcatOp should not be null.");
+ const size_t level = static_cast<size_t>(ctx->Attrs().Get<int>("level"));
+ const size_t axis = static_cast<size_t>(ctx->Attrs().Get<int>("axis"));
+ PADDLE_ENFORCE(level == 0UL || level == 1UL,
+ "The sequence_concat operator only accepts sequence "
+ "or a nested sequence as its input.");
+ auto ins_dims = ctx->GetInputsDim("X");
+ framework::DDim out_dims = ins_dims[0];
+ const size_t n = ins_dims.size();
+ for (size_t i = 1; i < n; ++i) {
+ out_dims[axis] += ins_dims[i][axis];
+ }
+ ctx->SetOutputDim("Out", out_dims);
+ }
+};
+
+class SequenceConcatOpMaker : public framework::OpProtoAndCheckerMaker {
+ public:
+ SequenceConcatOpMaker(framework::OpProto* proto,
+ framework::OpAttrChecker* op_checker)
+ : OpProtoAndCheckerMaker(proto, op_checker) {
+ AddInput("X",
+ "(A vector of LoDTensor), the input is a vector of LoDTensor, "
+ "each of which is a variable-length sequence or nested sequence.")
+ .AsDuplicable();
+ AddOutput("Out",
+ "(A LoDTensor), the variable-length output of "
+ "sequence_concat Op.");
+ AddAttr<int>("axis",
+ "(int, default 0)"
+ "The axis which the inputs will be joined with. "
+ "If axis is 0, the inputs will be joined with LoD index.")
+ .SetDefault(0);
+ AddAttr<int>("level",
+ "(int, default 0)"
+ "The level at which the inputs will be joined. "
+ "If the level is 0, the inputs will be joined at the nested "
+ "sequence level. "
+ "If the level is 1, the inputs will be joined at the "
+ "sequence level. "
+ "The level should be less than the level number of inputs.")
+ .SetDefault(0);
+ AddComment(R"DOC(
+ The sequence_concat operator concatenates multiple LoDTensors.
+ It only supports sequence (LoD Tensor with level number is 1)
+ or a nested sequence (LoD tensor with level number is 2) as its input.
+ - Case1:
+ If the axis is other than 0(here, axis is 1 and level is 1),
+ each input should have the same LoD information and the LoD
+ information of the output keeps the same as the input.
+
+ LoD(x0) = {{0,2,4}, {0,1,2,3,4}}; Dims(x0) = (4,3,4)
+ LoD(x1) = {{0,2,4}, {0,1,2,3,4}}; Dims(x1) = (4,4,4)
+ LoD(Out) = {{0,2,4}, {0,1,2,3,4}}; Dims(Out) = (4,7,4)
+
+ - Case2:
+ If the axis is 0(here, leve is 0), the inputs are concatenated along
+ time steps, the LoD information of the output need to re-compute.
+
+ LoD(x0) = {{0,2,4}, {0,1,2,3,4}}; Dims(x0) = (4,3,4)
+ LoD(x1) = {{0,3,5}, {0,1,2,3,5}}; Dims(x1) = (5,3,4)
+ LoD(Out) = {{0,5,9}, {0,1,2,3,4,5,6,7,9}}; Dims(Out) = (9,3,4)
+
+ - Case3:
+ If the axis is 0(here, level is 1).
+
+ LoD(x0) = {{0,2,4}, {0,1,2,3,4}}; Dims(x0) = (4,3,4)
+ LoD(x1) = {{0,3,5}, {0,1,3,4,5}}; Dims(x1) = (5,3,4)
+ LoD(Out) = {{0,5,9}, {0,2,5,7,9}}; Dims(Out) = (9,3,4)
+
+ NOTE: The levels of all the inputs should be the same.
+ )DOC");
+ }
+};
+
+class SequenceConcatGradOp : public framework::OperatorWithKernel {
+ public:
+ using framework::OperatorWithKernel::OperatorWithKernel;
+
+ protected:
+ void InferShape(framework::InferShapeContext* ctx) const override {
+ PADDLE_ENFORCE(ctx->HasInput(framework::GradVarName("Out")),
+ "The gradient of Out should not be null.");
+ PADDLE_ENFORCE(ctx->HasOutputs(framework::GradVarName("X")),
+ "The gradient of X should not be null.");
+ ctx->SetOutputsDim(framework::GradVarName("X"), ctx->GetInputsDim("X"));
+ }
+};
+
+} // namespace operators
+} // namespace paddle
+
+namespace ops = paddle::operators;
+REGISTER_OP(sequence_concat, ops::SequenceConcatOp, ops::SequenceConcatOpMaker,
+ sequence_concat_grad, ops::SequenceConcatGradOp);
+REGISTER_OP_CPU_KERNEL(
+ sequence_concat,
+ ops::SequenceConcatOpKernel<paddle::platform::CPUPlace, float>);
+REGISTER_OP_CPU_KERNEL(
+ sequence_concat_grad,
+ ops::SequenceConcatGradOpKernel<paddle::platform::CPUPlace, float>);
diff --git a/paddle/operators/sequence_concat_op.cu b/paddle/operators/sequence_concat_op.cu
new file mode 100644
index 0000000000..8dc4764785
--- /dev/null
+++ b/paddle/operators/sequence_concat_op.cu
@@ -0,0 +1,25 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#define EIGEN_USE_GPU
+
+#include "paddle/operators/sequence_concat_op.h"
+
+namespace ops = paddle::operators;
+REGISTER_OP_GPU_KERNEL(
+ sequence_concat,
+ ops::SequenceConcatOpKernel<paddle::platform::GPUPlace, float>);
+REGISTER_OP_GPU_KERNEL(
+ sequence_concat_grad,
+ ops::SequenceConcatGradOpKernel<paddle::platform::GPUPlace, float>);
diff --git a/paddle/operators/sequence_concat_op.h b/paddle/operators/sequence_concat_op.h
new file mode 100644
index 0000000000..a197a05bbb
--- /dev/null
+++ b/paddle/operators/sequence_concat_op.h
@@ -0,0 +1,155 @@
+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+#include "paddle/framework/op_registry.h"
+#include "paddle/operators/strided_memcpy.h"
+
+namespace paddle {
+namespace operators {
+
+using Tensor = framework::Tensor;
+using LoDTensor = framework::LoDTensor;
+using LoD = framework::LoD;
+
+template <typename T>
+LoD concatLoD(const std::vector<const T*> ins, const size_t axis,
+ const size_t level) {
+ auto out_lod = ins[0]->lod();
+ const size_t n = ins.size();
+ if (axis == 0UL) {
+ for (size_t i = 1; i < n; ++i) {
+ for (size_t j = 0; j < ins[i]->lod()[0].size(); ++j) {
+ out_lod[0][j] += ins[i]->lod()[0][j];
+ }
+
+ if (ins[0]->NumLevels() == 2) {
+ for (size_t j = 1; j < ins[i]->lod()[1].size(); ++j) {
+ if (level == 0UL) {
+ out_lod[1].push_back(out_lod[1].back() + ins[i]->lod()[1][j] -
+ ins[i]->lod()[1][j - 1]);
+ } else if (level == 1UL) {
+ out_lod[1][j] += ins[1]->lod()[1][j];
+ }
+ }
+ }
+ }
+ }
+ return out_lod;
+}
+
+template <typename Place, typename T>
+class SequenceConcatOpKernel : public framework::OpKernel<T> {
+ public:
+ void Compute(const framework::ExecutionContext& ctx) const override {
+ auto ins = ctx.MultiInput<LoDTensor>("X");
+ auto* out = ctx.Output<LoDTensor>("Out");
+ const size_t axis = static_cast<size_t>(ctx.Attr<int>("axis"));
+ const size_t level = static_cast<size_t>(ctx.Attr<int>("level"));
+ const size_t n = ins.size();
+
+ for (size_t i = 1; i < n; ++i) {
+ PADDLE_ENFORCE_EQ(ins[0]->NumLevels(), ins[i]->NumLevels(),
+ "The levels of all the input LoDTensors "
+ "should be the same.");
+ PADDLE_ENFORCE_EQ(ins[0]->dims().size(), ins[i]->dims().size(),
+ "The dimension size of all the input LoDTensors "
+ "should be the same.");
+
+ const size_t dims_size = ins[i]->dims().size();
+ for (size_t j = 0; j < dims_size; ++j) {
+ if (j == axis) continue;
+ PADDLE_ENFORCE_EQ(ins[0]->dims()[j], ins[i]->dims()[j],
+ "Except for the dimension of the specified "
+ "axis along which all the inputs are concatenated, "
+ "dimensions of all the other axises of the input "
+ "LoDTensors should be the same.");
+ }
+ }
+ PADDLE_ENFORCE_GT(ins[0]->NumLevels(), level,
+ "The levels of all the input LoDTensors "
+ "should be greater than the specify level");
+
+ out->mutable_data<T>(ctx.GetPlace());
+ auto out_lod = concatLoD<LoDTensor>(ins, axis, level);
+ out->set_lod(out_lod);
+
+ auto out_lod_level = out_lod[level];
+ for (size_t i = 0; i < out_lod_level.size() - 1; ++i) {
+ Tensor out_t = out->Slice<T>(static_cast<int>(out_lod_level[i]),
+ static_cast<int>(out_lod_level[i + 1]));
+ auto out_stride = framework::stride(out_t.dims());
+ size_t offset = 0;
+
+ for (size_t j = 0; j < n; ++j) {
+ auto in_lod_level = ins[j]->lod()[level];
+ auto in_stride = framework::stride(ins[j]->dims());
+ Tensor in_t = ins[j]->Slice<T>(static_cast<int>(in_lod_level[i]),
+ static_cast<int>(in_lod_level[i + 1]));
+ size_t axis_dim = in_t.dims()[axis];
+ StridedMemcpy<T>(ctx.device_context(), in_t.data<T>(), in_stride,
+ in_t.dims(), out_stride, out_t.data<T>() + offset);
+ offset += axis_dim * in_stride[axis];
+ }
+ }
+ }
+};
+
+template <typename Place, typename T>
+class SequenceConcatGradOpKernel : public framework::OpKernel<T> {
+ public:
+ void Compute(const framework::ExecutionContext& ctx) const override {
+ auto ins = ctx.MultiInput<framework::LoDTensor>("X");
+ auto* out_grad =
+ ctx.Input<framework::LoDTensor>(framework::GradVarName("Out"));
+ auto x_grads =
+ ctx.MultiOutput<framework::LoDTensor>(framework::GradVarName("X"));
+ size_t axis = static_cast<size_t>(ctx.Attr<int>("axis"));
+ size_t level = static_cast<size_t>(ctx.Attr<int>("level"));
+ const size_t n = x_grads.size();
+
+ // Set Grad(X) LoD as X
+ for (size_t i = 0; i < n; i++) {
+ x_grads[i]->set_lod(ins[i]->lod());
+ x_grads[i]->mutable_data<T>(ctx.GetPlace());
+ }
+
+ auto out_lod = concatLoD<LoDTensor>(ins, axis, level);
+ auto out_lod_level = out_lod[level];
+
+ for (size_t i = 0; i < out_lod_level.size() - 1; ++i) {
+ Tensor out_grad_t =
+ out_grad->Slice<T>(static_cast<int>(out_lod_level[i]),
+ static_cast<int>(out_lod_level[i + 1]));
+ auto out_grad_stride = framework::stride(out_grad_t.dims());
+ size_t offset = 0;
+
+ for (size_t j = 0; j < n; ++j) {
+ auto x_grad_lod_level = x_grads[j]->lod()[level];
+ auto x_grad_stride = framework::stride(x_grads[j]->dims());
+ Tensor x_grad_t =
+ x_grads[j]->Slice<T>(static_cast<int>(x_grad_lod_level[i]),
+ static_cast<int>(x_grad_lod_level[i + 1]));
+ size_t axis_dim = x_grad_t.dims()[axis];
+ StridedMemcpy<T>(ctx.device_context(), out_grad_t.data<T>() + offset,
+ out_grad_stride, out_grad_t.dims(), x_grad_stride,
+ x_grad_t.data<T>());
+ offset += axis_dim * out_grad_stride[axis];
+ }
+ }
+ }
+};
+
+} // namespace operators
+} // namespace paddle
diff --git a/paddle/operators/sum_op.cc b/paddle/operators/sum_op.cc
index ffb0cb9211..573487b835 100644
--- a/paddle/operators/sum_op.cc
+++ b/paddle/operators/sum_op.cc
@@ -34,7 +34,7 @@ class SumOp : public framework::OperatorWithKernel {
auto in_dim = x_dims[0];
for (size_t i = 1; i < N; i++) {
auto dim = x_dims[i];
- PADDLE_ENFORCE(in_dim == dim, "Input tensors must have same shape");
+ PADDLE_ENFORCE_EQ(in_dim, dim, "Input tensors must have same shape");
}
ctx->SetOutputDim("Out", in_dim);
ctx->ShareLoD("X", /*->*/ "Out");
diff --git a/paddle/operators/uniform_random_op.cc b/paddle/operators/uniform_random_op.cc
index e330877fc4..75928f1ec8 100644
--- a/paddle/operators/uniform_random_op.cc
+++ b/paddle/operators/uniform_random_op.cc
@@ -54,7 +54,7 @@ class UniformRandomOp : public framework::OperatorWithKernel {
PADDLE_ENFORCE(
ctx->Attrs().Get<float>("min") < ctx->Attrs().Get<float>("max"),
"uniform_random's min must less then max");
- auto dims = Attr<std::vector<int>>("dims");
+ auto& dims = ctx->Attrs().Get<std::vector<int>>("dims");
std::vector<int64_t> temp;
temp.reserve(dims.size());
for (auto dim : dims) {
diff --git a/paddle/platform/cudnn_helper.h b/paddle/platform/cudnn_helper.h
index 2841d2a2db..0c5719ef51 100644
--- a/paddle/platform/cudnn_helper.h
+++ b/paddle/platform/cudnn_helper.h
@@ -71,23 +71,32 @@ class ScopedTensorDescriptor {
inline cudnnTensorDescriptor_t descriptor(const cudnnTensorFormat_t format,
const cudnnDataType_t type,
- const std::vector<int>& dims) {
- // the format is not used now, but it maybe useful feature
+ const std::vector<int>& dims,
+ const int groups = 1) {
+ // the format is not used now, will add later
std::vector<int> strides(dims.size());
strides[dims.size() - 1] = 1;
for (int i = dims.size() - 2; i >= 0; i--) {
strides[i] = dims[i + 1] * strides[i + 1];
}
+ // Update tensor descriptor dims setting if groups > 1
+ // FIXME(typhoonzero): Assume using NCHW order
+ std::vector<int> dims_with_group(dims.begin(), dims.end()); // copy
+ if (groups > 1) {
+ dims_with_group[1] = dims_with_group[1] / groups;
+ }
PADDLE_ENFORCE(dynload::cudnnSetTensorNdDescriptor(
- desc_, type, dims.size(), dims.data(), strides.data()));
+ desc_, type, dims_with_group.size(), dims_with_group.data(),
+ strides.data()));
return desc_;
}
template <typename T>
inline cudnnTensorDescriptor_t descriptor(const DataLayout& order,
- const std::vector<int>& dims) {
- return descriptor(GetCudnnTensorFormat(order), CudnnDataType<T>::type,
- dims);
+ const std::vector<int>& dims,
+ const int groups = 1) {
+ return descriptor(GetCudnnTensorFormat(order), CudnnDataType<T>::type, dims,
+ groups);
}
private:
@@ -106,18 +115,29 @@ class ScopedFilterDescriptor {
inline cudnnFilterDescriptor_t descriptor(const cudnnTensorFormat_t format,
const cudnnDataType_t type,
- const std::vector<int>& kernel) {
- // filter layout: output input spatial_dim_y spatial_dim_x
+ const std::vector<int>& kernel,
+ const int groups = 1) {
+ // filter layout: MCHW, where M is the number of
+ // output image channels, C is the number of input image channels,
+ // H and W is height and width of filter.
+ std::vector<int> kernel_with_group(kernel.begin(), kernel.end());
+ if (groups > 1) {
+ // M /= groups
+ kernel_with_group[0] /= groups;
+ // NOTE: input filter(C) of the filter is already asserted to be C/groups.
+ }
PADDLE_ENFORCE(dynload::cudnnSetFilterNdDescriptor(
- desc_, type, format, kernel.size(), kernel.data()));
+ desc_, type, format, kernel_with_group.size(),
+ kernel_with_group.data()));
return desc_;
}
template <typename T>
inline cudnnFilterDescriptor_t descriptor(const DataLayout& order,
- const std::vector<int>& kernel) {
+ const std::vector<int>& kernel,
+ const int groups = 1) {
return descriptor(GetCudnnTensorFormat(order), CudnnDataType<T>::type,
- kernel);
+ kernel, groups);
}
private:
diff --git a/paddle/platform/gpu_info.cc b/paddle/platform/gpu_info.cc
index 70ad611d5d..0cab5ffc56 100644
--- a/paddle/platform/gpu_info.cc
+++ b/paddle/platform/gpu_info.cc
@@ -43,6 +43,8 @@ int GetCurrentDeviceId() {
}
void SetDeviceId(int id) {
+ // TODO(qijun): find a better way to cache the cuda device count
+ PADDLE_ENFORCE_LT(id, GetCUDADeviceCount(), "id must less than GPU count");
PADDLE_ENFORCE(cudaSetDevice(id),
"cudaSetDevice failed in paddle::platform::SetDeviceId");
}
diff --git a/paddle/pybind/CMakeLists.txt b/paddle/pybind/CMakeLists.txt
index 97364f2db9..b8fc934724 100644
--- a/paddle/pybind/CMakeLists.txt
+++ b/paddle/pybind/CMakeLists.txt
@@ -1,6 +1,6 @@
if(WITH_PYTHON)
cc_library(paddle_pybind SHARED
SRCS pybind.cc exception.cc protobuf.cc
- DEPS pybind python backward proto_desc tensor_array
+ DEPS pybind python backward proto_desc tensor_array paddle_memory
${GLOB_OP_LIB})
endif(WITH_PYTHON)
diff --git a/paddle/pybind/protobuf.cc b/paddle/pybind/protobuf.cc
index 116c99bd2c..7ab4e6a451 100644
--- a/paddle/pybind/protobuf.cc
+++ b/paddle/pybind/protobuf.cc
@@ -15,6 +15,7 @@ limitations under the License. */
#include "paddle/pybind/protobuf.h"
#include <deque>
#include <iostream>
+#include "paddle/framework/backward.h"
#include "paddle/framework/block_desc.h"
#include "paddle/framework/op_desc.h"
#include "paddle/framework/program_desc.h"
@@ -116,6 +117,11 @@ void BindProgramDesc(py::module &m) {
py::return_value_policy::reference)
.def("append_block", &ProgramDescBind::AppendBlock,
py::return_value_policy::reference)
+ .def("append_backward",
+ [](ProgramDescBind &program_desc,
+ const std::unordered_set<std::string> &no_grad_vars) {
+ AppendBackward(program_desc, no_grad_vars);
+ })
.def("block", &ProgramDescBind::Block, py::return_value_policy::reference)
.def("num_blocks", &ProgramDescBind::Size);
}
@@ -198,7 +204,8 @@ void BindOpDesc(py::module &m) {
.def("set_attr", &OpDescBind::SetAttr)
.def("attr", &OpDescBind::GetAttr)
.def("set_block_attr", &OpDescBind::SetBlockAttr)
- .def("get_block_attr", &OpDescBind::GetBlockAttr)
+ .def("block_attr", &OpDescBind::GetBlockAttr)
+ .def("check_attrs", &OpDescBind::CheckAttrs)
.def("infer_shape", &OpDescBind::InferShape);
}
diff --git a/paddle/pybind/pybind.cc b/paddle/pybind/pybind.cc
index 0f6e3101e2..cc9f7ffe02 100644
--- a/paddle/pybind/pybind.cc
+++ b/paddle/pybind/pybind.cc
@@ -18,6 +18,7 @@ limitations under the License. */
#include "paddle/framework/lod_tensor.h"
#include "paddle/framework/tensor_array.h"
#include "paddle/operators/cond_op.h"
+#include "paddle/operators/dynamic_recurrent_op.h"
#include "paddle/operators/net_op.h"
#include "paddle/operators/recurrent_op.h"
#include "paddle/platform/enforce.h"
@@ -341,6 +342,33 @@ All parameter, weight, gradient are variables in Paddle.
self.set_stepnet(net.Clone());
});
+ py::class_<operators::DynamicRecurrentOp, OperatorBase>(m,
+ "DynamicRecurrentOp")
+ .def_static("create",
+ [](py::bytes protobin) -> operators::DynamicRecurrentOp * {
+ OpDesc desc;
+ PADDLE_ENFORCE(desc.ParsePartialFromString(protobin),
+ "Cannot parse user input to OpDesc");
+ PADDLE_ENFORCE(desc.IsInitialized(),
+ "User OpDesc is not initialized, reason %s",
+ desc.InitializationErrorString());
+ auto rnn_op = OpRegistry::CreateOp(desc);
+ return static_cast<operators::DynamicRecurrentOp *>(
+ rnn_op.release());
+ })
+ .def("set_stepnet",
+ [](operators::DynamicRecurrentOp &self, const operators::NetOp &net)
+ -> void { self.SetStepNet(net.Clone()); })
+ .def("get_state",
+ [](operators::DynamicRecurrentOp &self, const std::string &name)
+ -> const TensorArray & { return self.state(name); })
+ .def("get_step_input",
+ [](operators::DynamicRecurrentOp &self, const std::string &name)
+ -> const TensorArray & { return self.step_input(name); })
+ .def("get_step_output",
+ [](operators::DynamicRecurrentOp &self, const std::string &name)
+ -> const TensorArray & { return self.step_output(name); });
+
// cond_op
py::class_<operators::CondOp, OperatorBase>(m, "CondOp")
.def_static("create",
diff --git a/paddle/pybind/tensor_py.h b/paddle/pybind/tensor_py.h
index 9e73f79cbd..85f9f22733 100644
--- a/paddle/pybind/tensor_py.h
+++ b/paddle/pybind/tensor_py.h
@@ -57,7 +57,18 @@ struct CastToPyBufferImpl<true, I, ARGS...> {
}
framework::Tensor dst_tensor;
if (paddle::platform::is_gpu_place(tensor.place())) {
- dst_tensor.CopyFrom<CUR_TYPE>(tensor, platform::CPUPlace());
+#ifdef PADDLE_WITH_CUDA
+ auto *src_ptr = static_cast<const void *>(tensor.data<CUR_TYPE>());
+ auto *dst_ptr = static_cast<void *>(dst_tensor.mutable_data<CUR_TYPE>(
+ tensor.dims(), platform::CPUPlace()));
+ // TODO(qijun): Here we use default CUDA stream to set GPU Tensor to
+ // a Python numpy array. It's better to manage CDUA stream unifiedly.
+ paddle::platform::GpuMemcpySync(dst_ptr, src_ptr,
+ sizeof(CUR_TYPE) * tensor.numel(),
+ cudaMemcpyDeviceToHost);
+#else
+ PADDLE_THROW("'GPUPlace' is not supported in CPU only device.");
+#endif
} else if (paddle::platform::is_cpu_place(tensor.place())) {
dst_tensor = tensor;
}
@@ -120,6 +131,8 @@ void PyCUDATensorSetFromArray(
self.Resize(framework::make_ddim(dims));
auto *dst = self.mutable_data<T>(place);
+ // TODO(qijun): Here we use default CUDA stream to set a Python numpy
+ // array to a GPU Tensor. It's better to manage CDUA stream unifiedly.
paddle::platform::GpuMemcpySync(dst, array.data(), sizeof(T) * array.size(),
cudaMemcpyHostToDevice);
}
diff --git a/proto/CMakeLists.txt b/proto/CMakeLists.txt
index 6212c2e60a..5d898d860c 100644
--- a/proto/CMakeLists.txt
+++ b/proto/CMakeLists.txt
@@ -1,4 +1,10 @@
-file(GLOB proto_filenames . *.proto)
+if (MOBILE_INFERENCE)
+ file(GLOB proto_filenames . ModelConfig.proto ParameterConfig.proto
+ TrainerConfig.proto DataConfig.proto)
+else()
+ file(GLOB proto_filenames . *.proto)
+endif()
+
include_directories(${CMAKE_CURRENT_BINARY_DIR})
proto_library(paddle_proto SRCS ${proto_filenames})
diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py
index d37f29d2c4..5043fb811d 100644
--- a/python/paddle/trainer_config_helpers/layers.py
+++ b/python/paddle/trainer_config_helpers/layers.py
@@ -318,7 +318,7 @@ class LayerOutput(object):
:param activation: Layer Activation.
:type activation: BaseActivation.
:param parents: Layer's parents.
- :type parents: list|tuple|collections.Sequence
+ :type parents: list | tuple | collections.Sequence
"""
def __init__(self,
@@ -435,7 +435,7 @@ def full_matrix_projection(input, size=0, param_attr=None):
size=100,
param_attr=ParamAttr(name='_proj'))
- :param input: input layer
+ :param input: The input of this layer.
:type input: LayerOutput
:param size: The parameter size. Means the width of parameter.
:type size: int
@@ -471,7 +471,7 @@ def trans_full_matrix_projection(input, size=0, param_attr=None):
initial_mean=0.0,
initial_std=0.01))
- :param input: input layer
+ :param input: The input of this layer.
:type input: LayerOutput
:param size: The parameter size. Means the width of parameter.
:type size: int
@@ -516,7 +516,7 @@ def table_projection(input, size=0, param_attr=None):
param_attr=ParamAttr(name='_proj'))
- :param input: Input layer, which must contains id fields.
+ :param input: The input of this layer, which must contains id fields.
:type input: LayerOutput
:param size: The parameter size. Means the width of parameter.
:type size: int
@@ -561,7 +561,7 @@ def identity_projection(input, offset=None, size=None):
Note that both of two projections should not have any parameter.
- :param input: Input Layer.
+ :param input: The input of this layer.
:type input: LayerOutput
:param offset: Offset, None if use default.
:type offset: int
@@ -596,7 +596,7 @@ def slice_projection(input, slices):
Note that slice_projection should not have any parameter.
- :param input: Input Layer.
+ :param input: The input of this layer.
:type input: LayerOutput
:param slices: An array of slice parameters.
Each slice contains the start and end offsets based
@@ -634,7 +634,7 @@ def scaling_projection(input, param_attr=None):
proj = scaling_projection(input=layer)
- :param input: Input Layer.
+ :param input: The input of this layer.
:type input: LayerOutput
:param param_attr: Parameter config, None if use default.
:type param_attr: ParameterAttribute
@@ -663,7 +663,7 @@ def dotmul_projection(input, param_attr=None):
proj = dotmul_projection(input=layer)
- :param input: Input layer.
+ :param input: The input of this layer.
:type input: LayerOutput
:param param_attr: Parameter config, None if use default.
:type param_attr: ParameterAttribute
@@ -734,7 +734,7 @@ def context_projection(input,
after context projection and not set padding_attr, sequence will
be [ 0AB ABC BCD CDE DEF EFG FG0 ].
- :param input: Input Sequence.
+ :param input: The input of this layer, which should be a sequence.
:type input: LayerOutput
:param context_len: context length.
:type context_len: int
@@ -744,7 +744,7 @@ def context_projection(input,
:param padding_attr: Padding Parameter Attribute. If false, it means padding
always be zero. Otherwise Padding is learnable, and
parameter attribute is set by this parameter.
- :type padding_attr: bool|ParameterAttribute
+ :type padding_attr: bool | ParameterAttribute
:return: Projection
:rtype: Projection
"""
@@ -782,13 +782,13 @@ class MixedLayerType(LayerOutput):
:type name: basestring
:param size: layer size.
:type size: int
- :param act: activation type.
+ :param act: Activation type.
:type act: BaseActivation
:param bias_attr: The Bias Attribute. If the parameter is set to
False or something not type of ParameterAttribute,
no bias is defined. If the parameter is set to
True, the bias is initialized to zero.
- :type bias_attr: ParameterAttribute|None|Bool|Any
+ :type bias_attr: ParameterAttribute | None | bool | Any
:param layer_attr: Extra Layer Attribute.
:type layer_attr: ExtraLayerAttribute or None
"""
@@ -880,15 +880,15 @@ def mixed_layer(size=0,
:type name: basestring
:param size: layer size.
:type size: int
- :param input: inputs layer. It is an optional parameter. If set,
+ :param input: The input of this layer. It is an optional parameter. If set,
then this function will just return layer's name.
- :param act: Activation Type.
+ :param act: Activation Type. LinearActivation is the default.
:type act: BaseActivation
:param bias_attr: The Bias Attribute. If the parameter is set to
False or something not type of ParameterAttribute,
no bias is defined. If the parameter is set to
True, the bias is initialized to zero.
- :type bias_attr: ParameterAttribute|None|Bool|Any
+ :type bias_attr: ParameterAttribute | None | bool | Any
:param layer_attr: The extra layer config. Default is None.
:type layer_attr: ExtraLayerAttribute
:return: MixedLayerType object can add inputs or layer name.
@@ -929,9 +929,9 @@ def data_layer(name, size, depth=None, height=None, width=None,
:param size: Size of this data layer.
:type size: int
:param height: Height of this data layer, used for image
- :type height: int|None
+ :type height: int | None
:param width: Width of this data layer, used for image
- :type width: int|None
+ :type width: int | None
:param layer_attr: Extra Layer Attribute.
:type layer_attr: ExtraLayerAttribute.
:return: LayerOutput object.
@@ -966,15 +966,15 @@ def embedding_layer(input, size, name=None, param_attr=None, layer_attr=None):
:param name: The name of this layer. It is optional.
:type name: basestring
- :param input: The input layer for this embedding. NOTE: must be Index Data.
+ :param input: The input of this layer, which must be Index Data.
:type input: LayerOutput
:param size: The embedding dimension.
:type size: int
:param param_attr: The embedding parameter attribute. See ParameterAttribute
for details.
- :type param_attr: ParameterAttribute|None
+ :type param_attr: ParameterAttribute | None
:param layer_attr: Extra layer Config. Default is None.
- :type layer_attr: ExtraLayerAttribute|None
+ :type layer_attr: ExtraLayerAttribute | None
:return: LayerOutput object.
:rtype: LayerOutput
"""
@@ -1021,11 +1021,11 @@ def fc_layer(input,
:param name: The name of this layer. It is optional.
:type name: basestring
- :param input: The input layer. Could be a list/tuple of input layer.
- :type input: LayerOutput|list|tuple
+ :param input: The input of this layer.
+ :type input: LayerOutput | list | tuple
:param size: The layer dimension.
:type size: int
- :param act: Activation Type. Default is tanh.
+ :param act: Activation Type. TanhActivation is the default.
:type act: BaseActivation
:param param_attr: The Parameter Attribute|list.
:type param_attr: ParameterAttribute
@@ -1033,9 +1033,9 @@ def fc_layer(input,
False or something not type of ParameterAttribute,
no bias is defined. If the parameter is set to
True, the bias is initialized to zero.
- :type bias_attr: ParameterAttribute|None|Bool|Any
+ :type bias_attr: ParameterAttribute | None | bool | Any
:param layer_attr: Extra Layer config.
- :type layer_attr: ExtraLayerAttribute|None
+ :type layer_attr: ExtraLayerAttribute | None
:return: LayerOutput object.
:rtype: LayerOutput
"""
@@ -1072,8 +1072,8 @@ def printer_layer(input, format=None, name=None):
:param name: The name of this layer. It is optional.
:type name: basestring
- :param input: The input layer. Could be a list/tuple of input layer.
- :type input: LayerOutput|list|tuple
+ :param input: The input of this layer.
+ :type input: LayerOutput | list | tuple
:return: LayerOutput
"""
if isinstance(input, LayerOutput):
@@ -1110,7 +1110,7 @@ def priorbox_layer(input,
:param name: The name of this layer. It is optional.
:type name: basestring
- :param input: The input layer.
+ :param input: The input of this layer.
:type input: LayerOutput
:param image: The network input image.
:type image: LayerOutput
@@ -1306,7 +1306,7 @@ def cross_channel_norm_layer(input, name=None, param_attr=None):
:param name: The name of this layer. It is optional.
:type name: basestring
- :param input: The input layer.
+ :param input: The input of this layer.
:type input: LayerOutput
:param param_attr: The Parameter Attribute|list.
:type param_attr: ParameterAttribute
@@ -1371,20 +1371,20 @@ def pooling_layer(input,
:type agg_level: AggregateLevel
:param name: The name of this layer. It is optional.
:type name: basestring
- :param input: input layer name.
+ :param input: The input of this layer.
:type input: LayerOutput
:param pooling_type: Type of pooling, MaxPooling(default), AvgPooling,
SumPooling, SquareRootNPooling.
- :type pooling_type: BasePoolingType|None
+ :type pooling_type: BasePoolingType | None
:param stride: The step size between successive pooling regions.
:type stride: Int
:param bias_attr: The Bias Attribute. If the parameter is set to
False or something not type of ParameterAttribute,
no bias is defined. If the parameter is set to
True, the bias is initialized to zero.
- :type bias_attr: ParameterAttribute|None|Bool|Any
+ :type bias_attr: ParameterAttribute | None | bool | Any
:param layer_attr: The Extra Attributes for layer, such as dropout.
- :type layer_attr: ExtraLayerAttribute|None
+ :type layer_attr: ExtraLayerAttribute | None
:return: LayerOutput object.
:rtype: LayerOutput
"""
@@ -1469,11 +1469,11 @@ def lstmemory(input,
:type name: basestring
:param size: DEPRECATED. size of the lstm cell
:type size: int
- :param input: input layer name.
+ :param input: The input of this layer.
:type input: LayerOutput
:param reverse: is sequence process reversed or not.
:type reverse: bool
- :param act: activation type, TanhActivation by default. :math:`h_t`
+ :param act: Activation type. TanhActivation is the default. :math:`h_t`
:type act: BaseActivation
:param gate_act: gate activation type, SigmoidActivation by default.
:type gate_act: BaseActivation
@@ -1483,11 +1483,11 @@ def lstmemory(input,
False or something not type of ParameterAttribute,
no bias is defined. If the parameter is set to
True, the bias is initialized to zero.
- :type bias_attr: ParameterAttribute|None|Bool|Any
+ :type bias_attr: ParameterAttribute | None | bool | Any
:param param_attr: Parameter Attribute.
- :type param_attr: ParameterAttribute|None|False
+ :type param_attr: ParameterAttribute | None | False
:param layer_attr: Extra Layer attribute
- :type layer_attr: ExtraLayerAttribute|None
+ :type layer_attr: ExtraLayerAttribute | None
:return: LayerOutput object.
:rtype: LayerOutput
"""
@@ -1591,14 +1591,14 @@ def grumemory(input,
gru = grumemory(input)
:param name: The gru layer name.
- :type name: None|basestring
- :param input: input layer.
+ :type name: None | basestring
+ :param input: The input of this layer.
:type input: LayerOutput.
:param size: DEPRECATED. size of the gru cell
:type size: int
:param reverse: Whether sequence process is reversed or not.
:type reverse: bool
- :param act: activation type, TanhActivation by default. This activation
+ :param act: Activation type, TanhActivation is the default. This activation
affects the :math:`{\\tilde{h_t}}`.
:type act: BaseActivation
:param gate_act: gate activation type, SigmoidActivation by default.
@@ -1609,11 +1609,11 @@ def grumemory(input,
False or something not type of ParameterAttribute,
no bias is defined. If the parameter is set to
True, the bias is initialized to zero.
- :type bias_attr: ParameterAttribute|None|Bool|Any
+ :type bias_attr: ParameterAttribute | None | bool | Any
:param param_attr: Parameter Attribute.
- :type param_attr: ParameterAttribute|None|False
+ :type param_attr: ParameterAttribute | None | False
:param layer_attr: Extra Layer attribute
- :type layer_attr: ExtraLayerAttribute|None
+ :type layer_attr: ExtraLayerAttribute | None
:return: LayerOutput object.
:rtype: LayerOutput
"""
@@ -1670,7 +1670,7 @@ def last_seq(input,
:param agg_level: Aggregated level
:param name: The name of this layer. It is optional.
:type name: basestring
- :param input: Input layer name.
+ :param input: The input of this layer.
:type input: LayerOutput
:param stride: The step size between successive pooling regions.
:type stride: Int
@@ -1726,7 +1726,7 @@ def first_seq(input,
:param agg_level: aggregation level
:param name: The name of this layer. It is optional.
:type name: basestring
- :param input: Input layer name.
+ :param input: The input of this layer.
:type input: LayerOutput
:param stride: The step size between successive pooling regions.
:type stride: Int
@@ -1799,7 +1799,7 @@ def expand_layer(input,
expand_as=layer2,
expand_level=ExpandLevel.FROM_NO_SEQUENCE)
- :param input: Input layer
+ :param input: The input of this layer.
:type input: LayerOutput
:param expand_as: Expand as this layer's sequence info.
:type expand_as: LayerOutput
@@ -1809,7 +1809,7 @@ def expand_layer(input,
False or something not type of ParameterAttribute,
no bias is defined. If the parameter is set to
True, the bias is initialized to zero.
- :type bias_attr: ParameterAttribute|None|Bool|Any
+ :type bias_attr: ParameterAttribute | None | bool | Any
:param expand_level: whether input layer is timestep(default) or sequence.
:type expand_level: ExpandLevel
:param layer_attr: extra layer attributes.
@@ -1858,7 +1858,7 @@ def repeat_layer(input,
expand = repeat_layer(input=layer, num_repeats=4)
- :param input: Input layer
+ :param input: The input of this layer.
:type input: LayerOutput
:param num_repeats: Repeat the input so many times
:type num_repeats: int
@@ -1869,7 +1869,7 @@ def repeat_layer(input,
False for treating input as column vector and repeating
in the row direction.
:type as_row_vector: bool
- :param act: Activation type.
+ :param act: Activation type. IdentityActivation is the default.
:type act: BaseActivation
:type name: basestring
:param layer_attr: extra layer attributes.
@@ -1917,13 +1917,13 @@ def seq_reshape_layer(input,
reshape = seq_reshape_layer(input=layer, reshape_size=4)
- :param input: Input layer.
+ :param input: The input of this layer.
:type input: LayerOutput
:param reshape_size: the size of reshaped sequence.
:type reshape_size: int
:param name: The name of this layer. It is optional.
:type name: basestring
- :param act: Activation type.
+ :param act: Activation type. IdentityActivation is the default.
:type act: BaseActivation
:param layer_attr: extra layer attributes.
:type layer_attr: ExtraLayerAttribute.
@@ -1931,7 +1931,7 @@ def seq_reshape_layer(input,
False or something not type of ParameterAttribute,
no bias is defined. If the parameter is set to
True, the bias is initialized to zero.
- :type bias_attr: ParameterAttribute|None|Bool|Any
+ :type bias_attr: ParameterAttribute | None | bool | Any
:return: LayerOutput object.
:rtype: LayerOutput
"""
@@ -1970,8 +1970,8 @@ def interpolation_layer(input, weight, name=None, layer_attr=None):
interpolation = interpolation_layer(input=[layer1, layer2], weight=layer3)
- :param input: Input layer.
- :type input: list|tuple
+ :param input: The input of this layer.
+ :type input: list | tuple
:param weight: Weight layer.
:type weight: LayerOutput
:param name: The name of this layer. It is optional.
@@ -2023,11 +2023,11 @@ def bilinear_interp_layer(input,
:param input: A input layer.
:type input: LayerOutput.
:param out_size_x: bilinear interpolation output width.
- :type out_size_x: int|None
+ :type out_size_x: int | None
:param out_size_y: bilinear interpolation output height.
- :type out_size_y: int|None
+ :type out_size_y: int | None
:param name: The layer's name, which cna not be specified.
- :type name: None|basestring
+ :type name: None | basestring
:param layer_attr: Extra Layer attribute.
:type layer_attr: ExtraLayerAttribute
:return: LayerOutput object.
@@ -2075,7 +2075,7 @@ def power_layer(input, weight, name=None, layer_attr=None):
power = power_layer(input=layer1, weight=layer2)
- :param input: Input layer.
+ :param input: The input of this layer.
:type input: LayerOutput
:param weight: Weight layer.
:type weight: LayerOutput
@@ -2119,7 +2119,7 @@ def scaling_layer(input, weight, name=None, layer_attr=None):
scale = scaling_layer(input=layer1, weight=layer2)
- :param input: Input layer.
+ :param input: The input of this layer.
:type input: LayerOutput
:param weight: Weight layer.
:type weight: LayerOutput
@@ -2159,7 +2159,7 @@ def trans_layer(input, name=None, layer_attr=None):
trans = trans_layer(input=layer)
- :param input: Input layer.
+ :param input: The input of this layer.
:type input: LayerOutput
:param name: The name of this layer. It is optional.
:type name: basestring
@@ -2197,7 +2197,7 @@ def rotate_layer(input, height, width, name=None, layer_attr=None):
height=100,
width=100)
- :param input: Input layer.
+ :param input: The input of this layer.
:type input: LayerOutput
:param height: The height of the sample matrix
:type height: int
@@ -2306,22 +2306,21 @@ def hsigmoid(input,
cost = hsigmoid(input=[layer1, layer2],
label=data_layer)
- :param input: Input layers. It could be a LayerOutput or list/tuple of
- LayerOutput.
- :type input: LayerOutput|list|tuple
+ :param input: The input of this layer.
+ :type input: LayerOutput | list | tuple
:param label: Label layer.
:type label: LayerOutput
:param num_classes: number of classes.
- :type num_classes: int|None
+ :type num_classes: int | None
:param name: The name of this layer. It is optional.
:type name: basestring
:param bias_attr: The Bias Attribute. If the parameter is set to
False or something not type of ParameterAttribute,
no bias is defined. If the parameter is set to
True, the bias is initialized to zero.
- :type bias_attr: ParameterAttribute|None|Bool|Any
+ :type bias_attr: ParameterAttribute | None | bool | Any
:param param_attr: Parameter Attribute. None means default parameter.
- :type param_attr: ParameterAttribute|None
+ :type param_attr: ParameterAttribute | None
:param layer_attr: Extra Layer Attribute.
:type layer_attr: ExtraLayerAttribute
:return: LayerOutput object.
@@ -2429,40 +2428,40 @@ def img_conv_layer(input,
:param name: The name of this layer. It is optional.
:type name: basestring
- :param input: Layer Input.
+ :param input: The input of this layer.
:type input: LayerOutput
:param filter_size: The x dimension of a filter kernel. Or input a tuple for
two image dimension.
- :type filter_size: int|tuple|list
+ :type filter_size: int | tuple | list
:param filter_size_y: The y dimension of a filter kernel. Since PaddlePaddle
currently supports rectangular filters, the filter's
shape will be (filter_size, filter_size_y).
- :type filter_size_y: int|None
+ :type filter_size_y: int | None
:param num_filters: Each filter group's number of filter
- :param act: Activation type. Default is tanh
+ :param act: Activation type. ReluActivation is the default.
:type act: BaseActivation
:param groups: Group size of filters.
:type groups: int
:param stride: The x dimension of the stride. Or input a tuple for two image
dimension.
- :type stride: int|tuple|list
+ :type stride: int | tuple | list
:param stride_y: The y dimension of the stride.
:type stride_y: int
:param padding: The x dimension of the padding. Or input a tuple for two
image dimension
- :type padding: int|tuple|list
+ :type padding: int | tuple | list
:param padding_y: The y dimension of the padding.
:type padding_y: int
:param dilation: The x dimension of the dilation. Or input a tuple for two
image dimension
- :type dilation: int|tuple|list
+ :type dilation: int | tuple | list
:param dilation_y: The y dimension of the dilation.
:type dilation_y: int
:param bias_attr: The Bias Attribute. If the parameter is set to
False or something not type of ParameterAttribute,
no bias is defined. If the parameter is set to
True, the bias is initialized to zero.
- :type bias_attr: ParameterAttribute|None|Bool|Any
+ :type bias_attr: ParameterAttribute | None | bool | Any
:param num_channels: number of input channels. If None will be set
automatically from previous output.
:type num_channels: int
@@ -2616,15 +2615,15 @@ def img_pool_layer(input,
:param padding: pooling padding width.
:type padding: int
:param padding_y: pooling padding height. It's equal to padding by default.
- :type padding_y: int|None
+ :type padding_y: int | None
:param name: name of pooling layer
:type name: basestring.
- :param input: layer's input
+ :param input: The input of this layer.
:type input: LayerOutput
:param pool_size: pooling window width
:type pool_size: int
:param pool_size_y: pooling window height. It's eaqual to pool_size by default.
- :type pool_size_y: int|None
+ :type pool_size_y: int | None
:param num_channels: number of input channel.
:type num_channels: int
:param pool_type: pooling type. MaxPooling or AvgPooling. Default is
@@ -2633,7 +2632,7 @@ def img_pool_layer(input,
:param stride: stride width of pooling.
:type stride: int
:param stride_y: stride height of pooling. It is equal to stride by default.
- :type stride_y: int|None
+ :type stride_y: int | None
:param layer_attr: Extra Layer attribute.
:type layer_attr: ExtraLayerAttribute
:param ceil_mode: Wether to use ceil mode to calculate output height and with.
@@ -2743,20 +2742,20 @@ def img_pool3d_layer(input,
pool_type=MaxPooling())
:param padding: pooling padding width.
- :type padding: int|tuple|list
+ :type padding: int | tuple | list
:param name: name of pooling layer
:type name: basestring.
- :param input: layer's input
+ :param input: The input of this layer.
:type input: LayerOutput
:param pool_size: pooling window width
- :type pool_size: int|tuple|list
+ :type pool_size: int | tuple | list
:param num_channels: number of input channel.
:type num_channels: int
:param pool_type: pooling type. MaxPooling or AvgPooling. Default is
MaxPooling.
:type pool_type: BasePoolingType
:param stride: stride width of pooling.
- :type stride: int|tuple|list
+ :type stride: int | tuple | list
:param layer_attr: Extra Layer attribute.
:type layer_attr: ExtraLayerAttribute
:param ceil_mode: Wether to use ceil mode to calculate output height and with.
@@ -2855,7 +2854,7 @@ def spp_layer(input,
:param name: The name of this layer. It is optional.
:type name: basestring
- :param input: layer's input.
+ :param input: The input of this layer.
:type input: LayerOutput
:param num_channels: number of input channel.
:type num_channels: int
@@ -2948,8 +2947,8 @@ def img_cmrnorm_layer(input,
norm = img_cmrnorm_layer(input=net, size=5)
:param name: The name of this layer. It is optional.
- :type name: None|basestring
- :param input: layer's input.
+ :type name: None | basestring
+ :param input: The input of this layer.
:type input: LayerOutput
:param size: Normalize in number of :math:`size` feature maps.
:type size: int
@@ -3024,7 +3023,7 @@ def batch_norm_layer(input,
batch_norm for CPU. Otherwise, select batch norm
type based on the specified type. If you use cudnn_batch_norm,
we suggested you use latest version, such as v5.1.
- :type batch_norm_type: None|string, None or "batch_norm" or "cudnn_batch_norm"
+ :type batch_norm_type: None | string, None or "batch_norm" or "cudnn_batch_norm"
:param act: Activation Type. Better be relu. Because batch
normalization will normalize input near zero.
:type act: BaseActivation
@@ -3034,7 +3033,7 @@ def batch_norm_layer(input,
:type num_channels: int
:param bias_attr: :math:`\\beta`, better be zero when initialize. So the
initial_std=0, initial_mean=1 is best practice.
- :type bias_attr: ParameterAttribute|None|Bool|Any
+ :type bias_attr: ParameterAttribute | None | bool | Any
:param param_attr: :math:`\\gamma`, better be one when initialize. So the
initial_std=0, initial_mean=1 is best practice.
:type param_attr: ParameterAttribute
@@ -3046,7 +3045,7 @@ def batch_norm_layer(input,
testing. If False, it will use the mean
and variance of current batch of test data for
testing.
- :type use_global_stats: bool|None.
+ :type use_global_stats: bool | None.
:param moving_average_fraction: Factor used in the moving average
computation, referred to as facotr,
:math:`runningMean = newMean*(1-factor)
@@ -3107,7 +3106,7 @@ def sum_to_one_norm_layer(input, name=None, layer_attr=None):
sum_to_one_norm = sum_to_one_norm_layer(input=layer)
- :param input: Input layer.
+ :param input: The input of this layer.
:type input: LayerOutput
:param name: The name of this layer. It is optional.
:type name: basestring
@@ -3143,7 +3142,7 @@ def row_l2_norm_layer(input, name=None, layer_attr=None):
row_l2_norm_layer = row_l2_norm_layer(input=layer)
- :param input: Input layer.
+ :param input: The input of this layer.
:type input: LayerOutput
:param name: The name of this layer. It is optional.
:type name: basestring
@@ -3201,14 +3200,14 @@ def addto_layer(input, act=None, name=None, bias_attr=None, layer_attr=None):
:type name: basestring
:param input: Input layers. It could be a LayerOutput or list/tuple of
LayerOutput.
- :type input: LayerOutput|list|tuple
- :param act: Activation Type, default is tanh.
+ :type input: LayerOutput | list | tuple
+ :param act: Activation Type. LinearActivation is the default.
:type act: BaseActivation
:param bias_attr: The Bias Attribute. If the parameter is set to
False or something not type of ParameterAttribute,
no bias is defined. If the parameter is set to
True, the bias is initialized to zero.
- :type bias_attr: ParameterAttribute|None|Bool|Any
+ :type bias_attr: ParameterAttribute | None | bool | Any
:param layer_attr: Extra Layer attribute.
:type layer_attr: ExtraLayerAttribute
:return: LayerOutput object.
@@ -3260,8 +3259,8 @@ def concat_layer(input, act=None, name=None, layer_attr=None, bias_attr=None):
:param name: The name of this layer. It is optional.
:type name: basestring
:param input: input layers or projections
- :type input: list|tuple|collections.Sequence
- :param act: Activation type.
+ :type input: list | tuple | collections.Sequence
+ :param act: Activation type. IdentityActivation is the default.
:type act: BaseActivation
:param layer_attr: Extra Layer Attribute.
:type layer_attr: ExtraLayerAttribute
@@ -3356,7 +3355,7 @@ def seq_concat_layer(a, b, act=None, name=None, layer_attr=None,
:type a: LayerOutput
:param b: input sequence layer
:type b: LayerOutput
- :param act: Activation type.
+ :param act: Activation type. IdentityActivation is the default.
:type act: BaseActivation
:param layer_attr: Extra Layer Attribute.
:type layer_attr: ExtraLayerAttribute
@@ -3364,7 +3363,7 @@ def seq_concat_layer(a, b, act=None, name=None, layer_attr=None,
False or something not type of ParameterAttribute,
no bias is defined. If the parameter is set to
True, the bias is initialized to zero.
- :type bias_attr: ParameterAttribute|None|Bool|Any
+ :type bias_attr: ParameterAttribute | None | bool | Any
:return: LayerOutput object.
:rtype: LayerOutput
"""
@@ -3440,9 +3439,9 @@ def memory(name,
:param is_seq: DEPRECATED. is sequence for boot_layer
:type is_seq: bool
:param boot_layer: boot layer of memory.
- :type boot_layer: LayerOutput|None
+ :type boot_layer: LayerOutput | None
:param boot_bias: boot layer's bias
- :type boot_bias: ParameterAttribute|None
+ :type boot_bias: ParameterAttribute | None
:param boot_bias_active_type: boot layer's active type.
:type boot_bias_active_type: BaseActivation
:param boot_with_const_id: boot layer's id.
@@ -3537,19 +3536,17 @@ def lstm_step_layer(input,
:type input: LayerOutput
:param state: State Layer. :math:`c_{t-1}`
:type state: LayerOutput
- :param act: Activation type. Default is tanh
+ :param act: Activation type. TanhActivation is the default.
:type act: BaseActivation
- :param gate_act: Gate Activation Type. Default is sigmoid, and should
- be sigmoid only.
+ :param gate_act: Gate Activation Type. SigmoidActivation is the default.
:type gate_act: BaseActivation
- :param state_act: State Activation Type. Default is sigmoid, and should
- be sigmoid only.
+ :param state_act: State Activation Type. TanhActivation is the default.
:type state_act: BaseActivation
:param bias_attr: The Bias Attribute. If the parameter is set to
False or something not type of ParameterAttribute,
no bias is defined. If the parameter is set to
True, the bias is initialized to zero.
- :type bias_attr: ParameterAttribute|None|Bool|Any
+ :type bias_attr: ParameterAttribute | None | bool | Any
:param layer_attr: layer's extra attribute.
:type layer_attr: ExtraLayerAttribute
:return: LayerOutput object.
@@ -3600,13 +3597,15 @@ def gru_step_layer(input,
:param output_mem:
:param size:
:param act:
+ :type act: BaseActivation
:param name: The name of this layer. It is optional.
- :param gate_act:
+ :param gate_act: Activation type of this layer's two gates. Default is Sigmoid.
+ :type gate_act: BaseActivation
:param bias_attr: The Bias Attribute. If the parameter is set to
False or something not type of ParameterAttribute,
no bias is defined. If the parameter is set to
True, the bias is initialized to zero.
- :type bias_attr: ParameterAttribute|None|Bool|Any
+ :type bias_attr: ParameterAttribute | None | bool | Any
:param param_attr: the parameter_attribute for transforming the output_mem
from previous step.
:param layer_attr:
@@ -3662,12 +3661,14 @@ def gru_step_naive_layer(input,
:param size:
:param name: The name of this layer. It is optional.
:param act:
- :param gate_act:
+ :type act: BaseActivation
+ :param gate_act: Activation type of this layer's two gates. Default is Sigmoid.
+ :type gate_act: BaseActivation
:param bias_attr: The Bias Attribute. If the parameter is set to
False or something not type of ParameterAttribute,
no bias is defined. If the parameter is set to
True, the bias is initialized to zero.
- :type bias_attr: ParameterAttribute|None|Bool|Any
+ :type bias_attr: ParameterAttribute | None | bool | Any
:param param_attr:
:param layer_attr:
:return:
@@ -3786,15 +3787,15 @@ def recurrent_layer(input,
out_{i} = act(in_{i} + out_{i+1} * W) \\ \\ \\text{for} \\ start <= i < end
- :param input: Input Layer
+ :param input: The input of this layer.
:type input: LayerOutput
- :param act: activation.
+ :param act: Activation type. TanhActivation is the default.
:type act: BaseActivation
:param bias_attr: The Bias Attribute. If the parameter is set to
False or something not type of ParameterAttribute,
no bias is defined. If the parameter is set to
True, the bias is initialized to zero.
- :type bias_attr: ParameterAttribute|None|Bool|Any
+ :type bias_attr: ParameterAttribute | None | bool | Any
:param param_attr: parameter attribute.
:type param_attr: ParameterAttribute
:param name: The name of this layer. It is optional.
@@ -3901,7 +3902,7 @@ def recurrent_group(step, input, reverse=False, name=None, targetInlink=None):
StaticInput will be imported to each time step, and doesn't change
through time. It's a mechanism to access layer outside step function.
- :type input: LayerOutput|StaticInput|SubsequenceInput|list|tuple
+ :type input: LayerOutput | StaticInput | SubsequenceInput | list | tuple
:param reverse: If reverse is set true, the recurrent unit will process the
input sequence in a reverse order.
@@ -3916,7 +3917,7 @@ def recurrent_group(step, input, reverse=False, name=None, targetInlink=None):
of words in each sentence) with all layer group's outputs.
targetInlink should be one of the layer group's input.
- :type targetInlink: LayerOutput|SubsequenceInput
+ :type targetInlink: LayerOutput | SubsequenceInput
:return: LayerOutput object.
:rtype: LayerOutput
@@ -4034,7 +4035,7 @@ def maxid_layer(input, name=None, layer_attr=None):
maxid = maxid_layer(input=layer)
- :param input: Input layer name.
+ :param input: The input of this layer.
:type input: LayerOutput
:param name: The name of this layer. It is optional.
:type name: basestring
@@ -4112,7 +4113,7 @@ def eos_layer(input, eos_id, name=None, layer_attr=None):
:param name: The name of this layer. It is optional.
:type name: basestring
- :param input: Input layer name.
+ :param input: The input of this layer.
:type input: LayerOutput
:param eos_id: end id of sequence
:type eos_id: int
@@ -4504,7 +4505,7 @@ def conv_projection(input,
num_filters=64,
num_channels=64)
- :param input: input layer
+ :param input: The input of this layer.
:type input: LayerOutput
:param filter_size: The x dimension of a filter kernel.
:type filter_size: int
@@ -4529,7 +4530,7 @@ def conv_projection(input,
:param param_attr: Convolution param attribute. None means default attribute
:type param_attr: ParameterAttribute
:param trans: whether it is convTrans or conv
- :type trans: boolean
+ :type trans: bool
:return: A DotMulProjection Object.
:rtype: DotMulProjection
"""
@@ -4637,14 +4638,14 @@ def pad_layer(input,
pad_h=[0,0],
pad_w=[2,2])
- :param input: layer's input.
+ :param input: The input of this layer.
:type input: LayerOutput
:param pad_c: padding size in channel dimension.
- :type pad_c: list|None
+ :type pad_c: list | None
:param pad_h: padding size in height dimension.
- :type pad_h: list|None
+ :type pad_h: list | None
:param pad_w: padding size in width dimension.
- :type pad_w: list|None
+ :type pad_w: list | None
:param layer_attr: Extra Layer Attribute.
:type layer_attr: ExtraLayerAttribute
:param name: The name of this layer. It is optional.
@@ -4779,7 +4780,7 @@ def tensor_layer(a,
:type b: LayerOutput
:param size: the layer dimension.
:type size: int.
- :param act: Activation Type. Default is tanh.
+ :param act: Activation type. LinearActivation is the default.
:type act: BaseActivation
:param param_attr: The Parameter Attribute.
:type param_attr: ParameterAttribute
@@ -4787,9 +4788,9 @@ def tensor_layer(a,
False or something not type of ParameterAttribute,
no bias is defined. If the parameter is set to
True, the bias is initialized to zero.
- :type bias_attr: ParameterAttribute|None|Bool|Any
+ :type bias_attr: ParameterAttribute | None | bool | Any
:param layer_attr: Extra Layer config.
- :type layer_attr: ExtraLayerAttribute|None
+ :type layer_attr: ExtraLayerAttribute | None
:return: LayerOutput object.
:rtype: LayerOutput
"""
@@ -4836,15 +4837,15 @@ def selective_fc_layer(input,
:param name: The name of this layer. It is optional.
:type name: basestring
- :param input: The input layer.
- :type input: LayerOutput|list|tuple
+ :param input: The input of this layer.
+ :type input: LayerOutput | list | tuple
:param select: The select layer. The output of select layer should be a
sparse binary matrix, and treat as the mask of selective fc.
If is None, acts exactly like fc_layer.
:type select: LayerOutput
:param size: The layer dimension.
:type size: int
- :param act: Activation Type. Default is tanh.
+ :param act: Activation type. TanhActivation is the default.
:type act: BaseActivation
:param param_attr: The Parameter Attribute.
:type param_attr: ParameterAttribute
@@ -4852,9 +4853,9 @@ def selective_fc_layer(input,
False or something not type of ParameterAttribute,
no bias is defined. If the parameter is set to
True, the bias is initialized to zero.
- :type bias_attr: ParameterAttribute|None|Bool|Any
+ :type bias_attr: ParameterAttribute | None | bool | Any
:param layer_attr: Extra Layer config.
- :type layer_attr: ExtraLayerAttribute|None
+ :type layer_attr: ExtraLayerAttribute | None
:return: LayerOutput object.
:rtype: LayerOutput
"""
@@ -4906,12 +4907,12 @@ def sampling_id_layer(input, name=None, layer_attr=None):
samping_id = sampling_id_layer(input=input)
- :param input: The input layer.
+ :param input: The input of this layer.
:type input: LayerOutput
:param name: The name of this layer. It is optional.
:type name: basestring
:param layer_attr: Extra Layer config.
- :type layer_attr: ExtraLayerAttribute|None
+ :type layer_attr: ExtraLayerAttribute | None
:return: LayerOutput object.
:rtype: LayerOutput
"""
@@ -4944,7 +4945,7 @@ def slope_intercept_layer(input,
scale = slope_intercept_layer(input=input, slope=-1.0, intercept=1.0)
- :param input: The input layer.
+ :param input: The input of this layer.
:type input: LayerOutput
:param name: The name of this layer. It is optional.
:type name: basestring
@@ -4953,7 +4954,7 @@ def slope_intercept_layer(input,
:param intercept: the offset.
:type intercept: float.
:param layer_attr: Extra Layer config.
- :type layer_attr: ExtraLayerAttribute|None
+ :type layer_attr: ExtraLayerAttribute | None
:return: LayerOutput object.
:rtype: LayerOutput
"""
@@ -5013,7 +5014,7 @@ def linear_comb_layer(weights, vectors, size=None, name=None, layer_attr=None):
:param name: The name of this layer. It is optional.
:type name: basestring
:param layer_attr: Extra Layer config.
- :type layer_attr: ExtraLayerAttribute|None
+ :type layer_attr: ExtraLayerAttribute | None
:return: LayerOutput object.
:rtype: LayerOutput
"""
@@ -5077,10 +5078,10 @@ def block_expand_layer(input,
block_x=1,
block_x=3)
- :param input: The input layer.
+ :param input: The input of this layer.
:type input: LayerOutput
:param num_channels: The channel number of input layer.
- :type num_channels: int|None
+ :type num_channels: int | None
:param block_x: The width of sub block.
:type block_x: int
:param block_y: The width of sub block.
@@ -5094,9 +5095,9 @@ def block_expand_layer(input,
:param padding_y: The padding size in vertical direction.
:type padding_y: int
:param name: The name of this layer. It is optional.
- :type name: None|basestring.
+ :type name: None | basestring.
:param layer_attr: Extra Layer config.
- :type layer_attr: ExtraLayerAttribute|None
+ :type layer_attr: ExtraLayerAttribute | None
:return: LayerOutput object.
:rtype: LayerOutput
"""
@@ -5155,15 +5156,15 @@ def maxout_layer(input, groups, num_channels=None, name=None, layer_attr=None):
num_channels=128,
groups=4)
- :param input: The input layer.
+ :param input: The input of this layer.
:type input: LayerOutput
:param num_channels: The channel number of input layer. If None will be set
automatically from previous output.
- :type num_channels: int|None
+ :type num_channels: int | None
:param groups: The group number of input layer.
:type groups: int
:param name: The name of this layer. It is optional.
- :type name: None|basestring.
+ :type name: None | basestring.
:param layer_attr: Extra Layer attribute.
:type layer_attr: ExtraLayerAttribute
:return: LayerOutput object.
@@ -5220,18 +5221,18 @@ def ctc_layer(input,
size=9055,
norm_by_times=True)
- :param input: The input layer.
+ :param input: The input of this layer.
:type input: LayerOutput
:param label: The data layer of label with variable length.
:type label: LayerOutput
:param size: category numbers + 1.
:type size: int
:param name: The name of this layer. It is optional.
- :type name: basestring|None
+ :type name: basestring | None
:param norm_by_times: Whether to normalization by times. False by default.
:type norm_by_times: bool
:param layer_attr: Extra Layer config.
- :type layer_attr: ExtraLayerAttribute|None
+ :type layer_attr: ExtraLayerAttribute | None
:return: LayerOutput object.
:rtype: LayerOutput
"""
@@ -5297,20 +5298,20 @@ def warp_ctc_layer(input,
blank=1000,
norm_by_times=False)
- :param input: The input layer.
+ :param input: The input of this layer.
:type input: LayerOutput
:param label: The data layer of label with variable length.
:type label: LayerOutput
:param size: category numbers + 1.
:type size: int
:param name: The name of this layer. It is optional.
- :type name: basestring|None
+ :type name: basestring | None
:param blank: the 'blank' label used in ctc
:type blank: int
:param norm_by_times: Whether to normalization by times. False by default.
:type norm_by_times: bool
:param layer_attr: Extra Layer config.
- :type layer_attr: ExtraLayerAttribute|None
+ :type layer_attr: ExtraLayerAttribute | None
:return: LayerOutput object.
:rtype: LayerOutput
"""
@@ -5368,11 +5369,11 @@ def crf_layer(input,
:param param_attr: Parameter attribute. None means default attribute
:type param_attr: ParameterAttribute
:param name: The name of this layer. It is optional.
- :type name: None|basestring
+ :type name: None | basestring
:param coeff: The coefficient affects the gradient in the backward.
:type coeff: float
:param layer_attr: Extra Layer config.
- :type layer_attr: ExtraLayerAttribute|None
+ :type layer_attr: ExtraLayerAttribute | None
:return: LayerOutput object.
:rtype: LayerOutput
"""
@@ -5438,9 +5439,9 @@ def crf_decoding_layer(input,
:param param_attr: Parameter attribute. None means default attribute
:type param_attr: ParameterAttribute
:param name: The name of this layer. It is optional.
- :type name: None|basestring
+ :type name: None | basestring
:param layer_attr: Extra Layer config.
- :type layer_attr: ExtraLayerAttribute|None
+ :type layer_attr: ExtraLayerAttribute | None
:return: LayerOutput object.
:rtype: LayerOutput
"""
@@ -5499,14 +5500,14 @@ def nce_layer(input,
:param name: The name of this layer. It is optional.
:type name: basestring
:param input: The input layers. It could be a LayerOutput of list/tuple of LayerOutput.
- :type input: LayerOutput|list|tuple|collections.Sequence
+ :type input: LayerOutput | list | tuple | collections.Sequence
:param label: label layer
:type label: LayerOutput
:param weight: weight layer, can be None(default)
:type weight: LayerOutput
:param num_classes: number of classes.
:type num_classes: int
- :param act: Activation, default is Sigmoid.
+ :param act: Activation type. SigmoidActivation is the default.
:type act: BaseActivation
:param param_attr: The Parameter Attribute|list.
:type param_attr: ParameterAttribute
@@ -5515,12 +5516,12 @@ def nce_layer(input,
:param neg_distribution: The distribution for generating the random negative labels.
A uniform distribution will be used if not provided.
If not None, its length must be equal to num_classes.
- :type neg_distribution: list|tuple|collections.Sequence|None
+ :type neg_distribution: list | tuple | collections.Sequence | None
:param bias_attr: The Bias Attribute. If the parameter is set to
False or something not type of ParameterAttribute,
no bias is defined. If the parameter is set to
True, the bias is initialized to zero.
- :type bias_attr: ParameterAttribute|None|Bool|Any
+ :type bias_attr: ParameterAttribute | None | bool | Any
:param layer_attr: Extra Layer Attribute.
:type layer_attr: ExtraLayerAttribute
:return: layer name.
@@ -5636,7 +5637,7 @@ def rank_cost(left,
It is an optional argument.
:type weight: LayerOutput
:param name: The name of this layer. It is optional.
- :type name: None|basestring
+ :type name: None | basestring
:param coeff: The coefficient affects the gradient in the backward.
:type coeff: float
:param layer_attr: Extra Layer Attribute.
@@ -5701,7 +5702,7 @@ def lambda_cost(input,
entire list of get gradient.
:type max_sort_size: int
:param name: The name of this layer. It is optional.
- :type name: None|basestring
+ :type name: None | basestring
:param layer_attr: Extra Layer Attribute.
:type layer_attr: ExtraLayerAttribute
:return: LayerOutput object.
@@ -5745,7 +5746,7 @@ def cross_entropy(input,
:param label: The input label.
:type input: LayerOutput.
:param name: The name of this layer. It is optional.
- :type name: None|basestring.
+ :type name: None | basestring.
:param coeff: The cost is multiplied with coeff.
The coefficient affects the gradient in the backward.
:type coeff: float.
@@ -5793,7 +5794,7 @@ def cross_entropy_with_selfnorm(input,
:param label: The input label.
:type input: LayerOutput.
:param name: The name of this layer. It is optional.
- :type name: None|basestring.
+ :type name: None | basestring.
:param coeff: The coefficient affects the gradient in the backward.
:type coeff: float.
:param softmax_selfnorm_alpha: The scale factor affects the cost.
@@ -5830,10 +5831,10 @@ def sum_cost(input, name=None, layer_attr=None):
cost = sum_cost(input=input_layer)
- :param input: The first input layer.
+ :param input: The input of this layer.
:type input: LayerOutput.
:param name: The name of this layer. It is optional.
- :type name: None|basestring.
+ :type name: None | basestring.
:param layer_attr: Extra Layer Attribute.
:type layer_attr: ExtraLayerAttribute
:return: LayerOutput object.
@@ -5878,7 +5879,7 @@ def huber_regression_cost(input,
:param label: The input label.
:type input: LayerOutput.
:param name: The name of this layer. It is optional.
- :type name: None|basestring.
+ :type name: None | basestring.
:param delta: The difference between the observed and predicted values.
:type delta: float.
:param coeff: The coefficient affects the gradient in the backward.
@@ -5928,7 +5929,7 @@ def huber_classification_cost(input,
:param label: The input label.
:type input: LayerOutput.
:param name: The name of this layer. It is optional.
- :type name: None|basestring.
+ :type name: None | basestring.
:param coeff: The coefficient affects the gradient in the backward.
:type coeff: float.
:param layer_attr: Extra Layer Attribute.
@@ -5971,7 +5972,7 @@ def multi_binary_label_cross_entropy(input,
:param label: The input label.
:type input: LayerOutput
:param name: The name of this layer. It is optional.
- :type name: None|basestring
+ :type name: None | basestring
:param coeff: The coefficient affects the gradient in the backward.
:type coeff: float
:param layer_attr: Extra Layer Attribute.
@@ -6139,7 +6140,7 @@ def smooth_l1_cost(input, label, name=None, coeff=1.0, layer_attr=None):
:param label: The input label.
:type input: LayerOutput
:param name: The name of this layer. It is optional.
- :type name: None|basestring
+ :type name: None | basestring
:param coeff: The coefficient affects the gradient in the backward.
:type coeff: float
:param layer_attr: Extra Layer Attribute.
@@ -6226,7 +6227,7 @@ def dropout_layer(input, dropout_rate, name=None):
:param name: The name of this layer. It is optional.
:type name: basestring
- :param input: The input layer.
+ :param input: The input of this layer.
:type input: LayerOutput
:param dropout_rate: The probability of dropout.
:type dropout_rate: float
@@ -6285,18 +6286,18 @@ def row_conv_layer(input,
row_conv = row_conv_layer(input=input_layer, context_len=3)
- :param input: The input layer.
+ :param input: The input of this layer.
:type input: LayerOutput
:param context_len: The context length equals the lookahead step number
plus one.
:type context_len: int
- :param act: Activation Type. Default is linear activation.
+ :param act: Activation Type. LinearActivation is the default.
:type act: BaseActivation
:param param_attr: The Parameter Attribute. If None, the parameter will be
initialized smartly. It's better to set it by yourself.
:type param_attr: ParameterAttribute
:param layer_attr: Extra Layer config.
- :type layer_attr: ExtraLayerAttribute|None
+ :type layer_attr: ExtraLayerAttribute | None
:return: LayerOutput object.
:rtype: LayerOutput
@@ -6342,7 +6343,7 @@ def prelu_layer(input,
:param name: The name of this layer. It is optional.
:type name: basestring
- :param input: The input layer.
+ :param input: The input of this layer.
:type input: LayerOutput
:param partial_sum: this parameter makes a group of inputs share a same weight.
@@ -6352,9 +6353,9 @@ def prelu_layer(input,
:type partial_sum: int
:param param_attr: The parameter attribute. See ParameterAttribute for details.
- :type param_attr: ParameterAttribute|None
+ :type param_attr: ParameterAttribute | None
:param layer_attr: Extra layer configurations. Default is None.
- :type layer_attr: ExtraLayerAttribute|None
+ :type layer_attr: ExtraLayerAttribute | None
:return: LayerOutput object.
:rtype: LayerOutput
"""
@@ -6407,37 +6408,37 @@ def gated_unit_layer(input,
.. code-block:: python
gated_unit = gated_unit_layer(size=128, input=input_layer))
- :param input: input for this layer.
+ :param input: The input of this layer.
:type input: LayerOutput
:param size: output size of the gated unit.
:type size: int
- :param act: activation type of the projected input.
+ :param act: Activation type of the projected input. LinearActivation is the default.
:type act: BaseActivation
:param name: The name of this layer. It is optional.
:type name: basestring
:param gate_attr: Attributes to tune the gate output, for example, error
clipping threshold, dropout and so on. See ExtraLayerAttribute for
more details.
- :type gate_attr: ExtraLayerAttribute|None
+ :type gate_attr: ExtraLayerAttribute | None
:param gate_param_attr: Attributes to tune the learnable projected matrix
parameter of the gate.
- :type gate_param_attr: ParameterAttribute|None
+ :type gate_param_attr: ParameterAttribute | None
:param gate_bias_attr: Attributes to tune the learnable bias of the gate.
- :type gate_bias_attr: ParameterAttribute|None
+ :type gate_bias_attr: ParameterAttribute | None
:param inproj_attr: Attributes to the tune the projected input, for
example, error clipping threshold, dropout and so on. See
ExtraLayerAttribute for more details.
- :type inproj_attr: ExtraLayerAttribute|None
+ :type inproj_attr: ExtraLayerAttribute | None
:param inproj_param_attr: Attributes to tune the learnable parameter of
the projection of input.
- :type inproj_param_attr: ParameterAttribute|None
+ :type inproj_param_attr: ParameterAttribute | None
:param inproj_bias_attr: Attributes to tune the learnable bias of
projection of the input.
- :type inproj_bias_attr: ParameterAttribute|None
+ :type inproj_bias_attr: ParameterAttribute | None
:param layer_attr: Attributes to tune the final output of the gated unit,
for example, error clipping threshold, dropout and so on. See
ExtraLayerAttribute for more details.
- :type layer_attr: ExtraLayerAttribute|None
+ :type layer_attr: ExtraLayerAttribute | None
:return: LayerOutput object.
:rtype: LayerOutput
"""
@@ -6487,7 +6488,7 @@ def switch_order_layer(input,
switch = switch_order(input=layer, name='switch', reshape_axis=reshape_axis)
reshape = {'height':[ 0, 1, 2], 'width':[3]}
- :param input: The input layer.
+ :param input: The input of this layer.
:type input: LayerOutput
:param name: The name of this layer. It is optional.
:type name: basestring
@@ -6521,7 +6522,7 @@ def switch_order_layer(input,
@layer_support()
def crop_layer(input, offset, axis=2, shape=None, name=None, layer_attr=None):
"""
- The crop layer crops images by offset and shape. User can set crop shape by
+ This layer crops images by offset and shape. User can set crop shape by
args 'shape' explicitly or by reference input layer.
The example usage is:
@@ -6529,10 +6530,10 @@ def crop_layer(input, offset, axis=2, shape=None, name=None, layer_attr=None):
.. code-block:: python
crop = crop_layer(input=[image_input, reference_input], axis=2, offset=[2, 3])
- :param input: The input layer.If two inputs were setted,
- the second input will be regarded as reference input
- :type input: LayerOutput or Sequence
- :param offset: The crop offset
+ :param input: The input of this layer. If two inputs are given, the second input
+ will be regarded as reference input.
+ :type input: LayerOutput | Sequence
+ :param offset: The crop offset.
:type offset: Sequence
:param axis: start axis to be cropped. To image input layer:
- 0: batch size
@@ -6581,12 +6582,12 @@ def sub_nested_seq_layer(input, selected_indices, name=None):
.. code-block:: python
- sub_nest_seq = sub_nested_seq_layer(input=[data, selected_indices])
+ sub_nest_seq = sub_nested_seq_layer(input=data, selected_indices=selected_ids)
- :param input: A nested sequence.
+ :param input: The input of this layer. It is a nested sequence.
:type input: LayerOutput
- :param selected_indices: a set of sequence indices in the nested sequence.
+ :param selected_indices: A set of sequence indices in the nested sequence.
:type input: LayerOutput
:param name: The name of this layer. It is optional.
:type name: basestring
@@ -6628,7 +6629,7 @@ def clip_layer(input, min, max, name=None):
:param name: The name of this layer. It is optional.
:type name: basestring
- :param input: The input layer.
+ :param input: The input of this layer.
:type input: LayerOutput.
:param min: The lower threshold for clipping.
:type min: double
@@ -6673,12 +6674,12 @@ def seq_slice_layer(input, starts, ends, name=None):
:param name: The name of this layer. It is optional.
:type name: basestring
- :param input: input for this layer, it should be a sequence.
+ :param input: The input of this layer, which should be a sequence.
:type input: LayerOutput
:param starts: start indices to slice the input sequence.
- :type starts: LayerOutput|None
+ :type starts: LayerOutput | None
:param ends: end indices to slice the input sequence.
- :type ends: LayerOutput|None
+ :type ends: LayerOutput | None
:return: LayerOutput object.
:rtype: LayerOutput
@@ -6727,9 +6728,9 @@ def kmax_seq_score_layer(input, name=None, beam_size=1):
:param name: The name of this layer. It is optional.
:type name: basestring
- :param input: The input layer. It stores scores over a sequence or a nested
+ :param input: The input of this layer. It stores scores over a sequence or a nested
sequence and its size must be 1.
- :type input: LayerOutput.
+ :type input: LayerOutput
:param beam_size: sequence indices with top beam_size scores are returned.
:type beam_size: double
:return: LayerOutput object.
@@ -6785,24 +6786,24 @@ def img_conv3d_layer(input,
:param name: The name of this layer. It is optional.
:type name: basestring
- :param input: Layer Input.
+ :param input: The input of this layer.
:type input: LayerOutput
:param filter_size: The x dimension of a filter kernel. Or input a list.
- :type filter_size: int|tuple|list
+ :type filter_size: int | tuple | list
:param num_filters: Each filter group's number of filter
- :param act: Activation type. Default is tanh
+ :param act: Activation type. ReluActivation is the default.
:type act: BaseActivation
:param groups: Group size of filters.
:type groups: int
:param stride: The x dimension of the stride. Or input a tuple for two image
dimension.
- :type stride: int|tuple|list
+ :type stride: int | tuple | list
:param padding: The x dimension of the padding. Or input a tuple for two
image dimension
- :type padding: int|tuple|list
+ :type padding: int | tuple | list
:param bias_attr: Convolution bias attribute. None means default bias.
False means no bias.
- :type bias_attr: ParameterAttribute|None|Bool|Any
+ :type bias_attr: ParameterAttribute | None | bool | Any
:param num_channels: number of input channels. If None will be set
automatically from previous output.
:type num_channels: int
@@ -6916,15 +6917,15 @@ def scale_shift_layer(input, name=None, param_attr=None, bias_attr=None):
:param name: The name of this layer. It is optional.
:type name: basestring
- :param input: The input layer.
- :type input: LayerOutput.
+ :param input: The input of this layer.
+ :type input: LayerOutput
:param param_attr: The parameter attribute of scaling.
:type param_attr: ParameterAttribute
:param bias_attr: The Bias Attribute. If the parameter is set to
False or something not type of ParameterAttribute,
no bias is defined. If the parameter is set to
True, the bias is initialized to zero.
- :type bias_attr: ParameterAttribute|None|Bool|Any
+ :type bias_attr: ParameterAttribute | None | bool | Any
:return: LayerOutput object.
:rtype: LayerOutput
"""
@@ -6944,11 +6945,11 @@ def resize_layer(input, size, name=None):
into the output matrix with a shape of [Height x Width / size, size],
where size is the parameter of this layer indicating the output dimension.
- :param input: The input to this layer.
+ :param input: The input of this layer.
:type input: LayerOutput.
:param name: The name of this layer. It is optional.
:type name: basestring
- :param size: The resized output dimesion of this layer.
+ :param size: The resized output dimension of this layer.
:type size: int
:return: A LayerOutput object.
:rtype: LayerOutput
diff --git a/python/paddle/v2/framework/graph.py b/python/paddle/v2/framework/framework.py
similarity index 66%
rename from python/paddle/v2/framework/graph.py
rename to python/paddle/v2/framework/framework.py
index 0f0a2847e5..2afbd0c831 100644
--- a/python/paddle/v2/framework/graph.py
+++ b/python/paddle/v2/framework/framework.py
@@ -1,4 +1,5 @@
import paddle.v2.framework.core as core
+import paddle.v2.framework.proto.framework_pb2 as framework_pb2
import collections
import numpy as np
import copy
@@ -106,6 +107,40 @@ class Variable(object):
raise ValueError("Not supported numpy dtype " + str(dtype))
+def get_all_op_protos():
+ """
+ Get all registered op proto from PaddlePaddle C++ end.
+ :return: A list of registered OpProto.
+ """
+ protostrs = core.get_all_op_protos()
+ ret_values = []
+ for pbstr in protostrs:
+ op_proto = framework_pb2.OpProto.FromString(str(pbstr))
+ ret_values.append(op_proto)
+ return ret_values
+
+
+class OpProtoHolder(object):
+ @classmethod
+ def instance(cls):
+ if not hasattr(cls, '_instance'):
+ cls._instance = cls()
+ return cls._instance
+
+ def __init__(self):
+ assert not hasattr(
+ self.__class__,
+ '_instance'), 'Please use `instance()` to get OpProtoHolder opject!'
+ op_protos = get_all_op_protos()
+ self.op_proto_map = {}
+ for proto in op_protos:
+ self.op_proto_map[proto.type] = proto
+
+ def get_op_proto(self, type):
+ assert type in self.op_proto_map, "Operator \"%s\" has not been registered." % type
+ return self.op_proto_map[type]
+
+
class Operator(object):
def __init__(self,
block,
@@ -116,20 +151,89 @@ class Operator(object):
attrs=None):
self.block = block
self.desc = desc
- if type is not None:
- # TODO.
- pass
+ if len(self.desc.type()) != 0:
+ return
+ if type is None:
+ raise ValueError(
+ "`type` to initilized an Operator can not be None.")
+ self.desc.set_type(type)
+ proto = OpProtoHolder.instance().get_op_proto(type)
+
if inputs is not None:
- # TODO
- pass
+ for in_proto in proto.inputs:
+ in_argus = inputs[in_proto.name]
+ if not isinstance(in_argus, list):
+ in_argus = [in_argus]
+ if not in_proto.duplicable and len(in_argus) > 1:
+ raise ValueError(
+ "Input %s expects only one input, but %d are given." %
+ (in_proto.name, len(in_argus)))
+ in_argu_names = []
+ for argu in in_argus:
+ in_argu_names.append(argu.name)
+ self.desc.set_input(in_proto.name, in_argu_names)
+
if outputs is not None:
- # TODO
- pass
+ for out_proto in proto.outputs:
+ out_argus = outputs[out_proto.name]
+ if not isinstance(out_argus, list):
+ out_argus = [out_argus]
+ if not out_proto.duplicable and len(out_argus) > 1:
+ raise ValueError(
+ "Output %s expects only one output, but %d are given." %
+ (out_proto.name, len(out_argus)))
+ out_argu_names = []
+ for argu in out_argus:
+ out_argu_names.append(argu.name)
+ argu.op = self
+ self.desc.set_output(out_proto.name, out_argu_names)
+
if attrs is not None:
- # TODO
- pass
+ for attr in proto.attrs:
+ attr_name = attr.name
+ if not attr_name in attrs:
+ continue
+ if not isinstance(attrs[attr_name], Block):
+ self.desc.set_attr(attr_name, attrs[attr_name])
+ else:
+ self.desc.set_block_attr(attr_name, attrs[attr_name].desc)
+
+ self.desc.check_attrs()
+ self.desc.infer_shape(self.block.desc)
+
+ @property
+ def type(self):
+ return self.desc.type()
+
+ def input(self, name):
+ return self.desc.input(name)
+
+ @property
+ def input_names(self):
+ return self.desc.input_names()
+
+ def output(self, name):
+ return self.desc.output(name)
+
+ @property
+ def output_names(self):
+ return self.desc.output_names()
+
+ def has_attr(self, name):
+ return self.desc.has_attr(name)
+
+ def attr_type(self, name):
+ return self.desc.attr_type(name)
+
+ @property
+ def attr_names(self):
+ return self.desc.attr_names()
+
+ def attr(self, name):
+ return self.desc.attr(name)
- # TODO: Getters
+ def block_attr(self, name):
+ return self.desc.block_attr(name)
class Block(object):
diff --git a/python/paddle/v2/framework/op.py b/python/paddle/v2/framework/op.py
index 9086a5cc34..bc771a964a 100644
--- a/python/paddle/v2/framework/op.py
+++ b/python/paddle/v2/framework/op.py
@@ -219,6 +219,27 @@ class __RecurrentOp__(object):
return core.RecurrentOp.create(proto.SerializeToString())
+class __DynamicRecurrentOp__(object):
+ __proto__ = None
+ type = "dynamic_recurrent"
+
+ def __init__(self):
+ # cache recurrent_op's proto
+ if self.__proto__ is None:
+ for op_proto in get_all_op_protos():
+ if op_proto.type == self.type:
+ self.__proto__ = op_proto
+
+ def __call__(self, *args, **kwargs):
+ if self.type not in args and "type" not in kwargs:
+ kwargs["type"] = self.type
+ # create proto
+ create_method = OpDescCreationMethod(self.__proto__)
+ proto = create_method(*args, **kwargs)
+ # create rnnop
+ return core.DynamicRecurrentOp.create(proto.SerializeToString())
+
+
class __CondOp__(object):
__proto__ = None
type = "cond"
@@ -242,4 +263,5 @@ class __CondOp__(object):
Operator = OperatorFactory() # The default global factory
RecurrentOp = __RecurrentOp__()
+DynamicRecurrentOp = __DynamicRecurrentOp__()
CondOp = __CondOp__()
diff --git a/python/paddle/v2/framework/tests/test_activation_op.py b/python/paddle/v2/framework/tests/test_activation_op.py
index 9157e00f6e..5831b880e4 100644
--- a/python/paddle/v2/framework/tests/test_activation_op.py
+++ b/python/paddle/v2/framework/tests/test_activation_op.py
@@ -78,6 +78,26 @@ class TestTanhShrink(OpTest):
self.check_grad(['X'], 'Y', max_relative_error=0.008)
+class TestHardShrink(OpTest):
+ def setUp(self):
+ self.op_type = "hard_shrink"
+ x = np.random.uniform(-1, 1, [4, 4]).astype("float32")
+ threshold = 0.5
+
+ self.inputs = {'X': x}
+ self.attrs = {'lambda': threshold}
+
+ t = np.copy(x)
+ t[(t >= -threshold) & (t <= threshold)] = 0
+ self.outputs = {'Y': t}
+
+ def test_check_output(self):
+ self.check_output()
+
+ def test_check_grad(self):
+ self.check_grad(['X'], 'Y', max_relative_error=0.005)
+
+
class TestSoftShrink(OpTest):
def setUp(self):
self.op_type = "softshrink"
@@ -311,6 +331,21 @@ class TestSTanh(OpTest):
self.check_grad(['X'], 'Y', max_relative_error=0.007)
+class TestSoftplus(OpTest):
+ def setUp(self):
+ self.op_type = "softplus"
+ self.inputs = {
+ 'X': np.random.uniform(-1, 1, [11, 17]).astype("float32")
+ }
+ self.outputs = {'Y': np.log(1 + np.exp(self.inputs['X']))}
+
+ def test_check_output(self):
+ self.check_output()
+
+ def test_check_grad(self):
+ self.check_grad(['X'], 'Y', max_relative_error=0.007)
+
+
class TestSoftsign(OpTest):
def setUp(self):
self.op_type = "softsign"
@@ -328,5 +363,54 @@ class TestSoftsign(OpTest):
self.check_grad(['X'], 'Y', max_relative_error=0.007)
+class TestThresholdedRelu(OpTest):
+ def setUp(self):
+ self.op_type = "thresholded_relu"
+ threshold = 0.25
+ self.relative_error = 0.005
+ X = np.random.uniform(-1, 1, [11, 17]).astype("float32")
+
+ # Same reason as TestAbs
+ X[np.abs(X - threshold) < self.relative_error] = threshold + 0.2
+
+ self.inputs = {'X': X}
+ self.attrs = {'threshold': threshold}
+ self.outputs = {'Y': (X > threshold) * X}
+
+ def test_check_output(self):
+ self.check_output()
+
+ def test_check_grad(self):
+ self.check_grad(['X'], 'Y', max_relative_error=self.relative_error)
+
+
+class TestHardSigmoid(OpTest):
+ def setUp(self):
+ self.op_type = "hard_sigmoid"
+ self.relative_error = 0.002
+
+ X = np.random.uniform(-5, 5, [2, 2]).astype("float32")
+ slope = 0.2
+ offset = 0.5
+ lower_threshold = -offset / slope
+ upper_threshold = (1 - offset) / slope
+
+ self.inputs = {'X': X}
+ # Same reason as TestAbs
+ X[np.abs(X - lower_threshold) < self.relative_error] = \
+ lower_threshold + 0.2
+ X[np.abs(X - upper_threshold) < self.relative_error] = \
+ upper_threshold - 0.2
+
+ temp = X * slope + offset
+ self.outputs = {'Y': np.maximum(0.0, np.minimum(1.0, temp))}
+
+ def test_check_output(self):
+ self.check_output()
+
+ def test_check_grad(self):
+ self.check_grad(['X'], 'Y', max_relative_error=0.002)
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/python/paddle/v2/framework/tests/test_adam_op.py b/python/paddle/v2/framework/tests/test_adam_op.py
new file mode 100644
index 0000000000..ff6faafa6e
--- /dev/null
+++ b/python/paddle/v2/framework/tests/test_adam_op.py
@@ -0,0 +1,186 @@
+import unittest
+import numpy as np
+from op_test import OpTest
+
+
+class TestAdamOp1(OpTest):
+ def setUp(self):
+ '''Test Adam Op with supplied attributes
+ '''
+ self.op_type = "adam"
+ param = np.random.uniform(-1, 1, (102, 105)).astype("float32")
+ grad = np.random.uniform(-1, 1, (102, 105)).astype("float32")
+ moment1 = np.random.uniform(-1, 1, (102, 105)).astype("float32")
+ # The second moment is positive
+ moment2 = np.random.random((102, 105)).astype("float32")
+
+ learning_rate = 0.004
+ beta1 = 0.78
+ beta2 = 0.836
+ epsilon = 1e-4
+ beta1_pow = beta1**10
+ beta2_pow = beta2**10
+
+ self.inputs = {
+ 'Param': param,
+ 'Grad': grad,
+ 'Moment1': moment1,
+ 'Moment2': moment2,
+ 'LearningRate': np.array([learning_rate]).astype("float32"),
+ 'Beta1Pow': np.array([beta1_pow]).astype("float32"),
+ 'Beta2Pow': np.array([beta2_pow]).astype("float32")
+ }
+
+ self.attrs = {'epsilon': epsilon, 'beta1': beta1, 'beta2': beta2}
+
+ param_out, moment1_out, moment2_out, beta1_pow_out, \
+ beta2_pow_out = adam_step(self.inputs, self.attrs)
+
+ self.outputs = {
+ 'Moment1Out': moment1_out,
+ 'Moment2Out': moment2_out,
+ 'Beta1PowOut': beta1_pow_out,
+ 'Beta2PowOut': beta2_pow_out,
+ 'ParamOut': param_out
+ }
+
+ def test_check_output(self):
+ self.check_output()
+
+
+class TestAdamOp2(OpTest):
+ def setUp(self):
+ '''Test Adam Op with supplied attributes
+ '''
+ self.op_type = "adam"
+ param = np.random.uniform(-1, 1, (102, 105)).astype("float32")
+ grad = np.random.uniform(-1, 1, (102, 105)).astype("float32")
+ moment1 = np.random.uniform(-1, 1, (102, 105)).astype("float32")
+ # The second moment is positive
+ moment2 = np.random.random((102, 105)).astype("float32")
+
+ learning_rate = 0.001
+ beta1 = 0.9
+ beta2 = 0.999
+ epsilon = 1e-8
+ beta1_pow = beta1**10
+ beta2_pow = beta2**10
+
+ self.inputs = {
+ 'Param': param,
+ 'Grad': grad,
+ 'Moment1': moment1,
+ 'Moment2': moment2,
+ 'LearningRate': np.array([learning_rate]).astype("float32"),
+ 'Beta1Pow': np.array([beta1_pow]).astype("float32"),
+ 'Beta2Pow': np.array([beta2_pow]).astype("float32")
+ }
+
+ attributes = {'epsilon': epsilon, 'beta1': beta1, 'beta2': beta2}
+
+ param_out, moment1_out, moment2_out, beta1_pow_out, \
+ beta2_pow_out = adam_step(self.inputs, attributes)
+
+ self.outputs = {
+ 'Moment1Out': moment1_out,
+ 'Moment2Out': moment2_out,
+ 'Beta1PowOut': beta1_pow_out,
+ 'Beta2PowOut': beta2_pow_out,
+ 'ParamOut': param_out
+ }
+
+ def test_check_output(self):
+ self.check_output()
+
+
+class TestAdamOpMultipleSteps(OpTest):
+ def setUp(self):
+ '''Test Adam Operator with supplied attributes
+ '''
+ self.op_type = "adam"
+ self.num_steps = 10
+
+ param = np.random.uniform(-1, 1, (102, 105)).astype("float32")
+ grad = np.random.uniform(-1, 1, (102, 105)).astype("float32")
+ moment1 = np.random.uniform(-1, 1, (102, 105)).astype("float32")
+ # The second moment is positive
+ moment2 = np.random.random((102, 105)).astype("float32")
+
+ learning_rate = 0.001
+ beta1 = 0.9
+ beta2 = 0.999
+ epsilon = 1e-8
+ beta1_pow = beta1**10
+ beta2_pow = beta2**10
+
+ self.inputs = {
+ 'Param': param,
+ 'Grad': grad,
+ 'Moment1': moment1,
+ 'Moment2': moment2,
+ 'LearningRate': np.array([learning_rate]).astype("float32"),
+ 'Beta1Pow': np.array([beta1_pow]).astype("float32"),
+ 'Beta2Pow': np.array([beta2_pow]).astype("float32")
+ }
+
+ self.attrs = {'epsilon': epsilon, 'beta1': beta1, 'beta2': beta2}
+
+ def test_check_output(self):
+ for _ in range(self.num_steps):
+ param_out, moment1_out, moment2_out, beta1_pow_out, \
+ beta2_pow_out = adam_step(self.inputs, self.attrs)
+
+ self.outputs = {
+ 'Moment1Out': moment1_out,
+ 'Moment2Out': moment2_out,
+ 'Beta1PowOut': beta1_pow_out,
+ 'Beta2PowOut': beta2_pow_out,
+ 'ParamOut': param_out
+ }
+
+ # Verify output for this step
+ self.check_output()
+
+ # Output of this step becomes input for next step
+ self.inputs['Param'] = param_out
+ self.inputs['Moment1'] = moment1_out
+ self.inputs['Moment2'] = moment2_out
+ self.inputs['Beta1Pow'] = beta1_pow_out
+ self.inputs['Beta2Pow'] = beta2_pow_out
+
+ # Randomize gradient for next step
+ self.inputs['Grad'] = np.random.uniform(
+ -1, 1, (102, 105)).astype("float32")
+
+
+def adam_step(inputs, attributes):
+ '''
+ Simulate one step of the adam optimizer
+ :param inputs: dict of inputs
+ :param attributes: dict of attributes
+ :return tuple: tuple of output param, moment1, moment2,
+ beta1 power accumulator and beta2 power accumulator
+ '''
+ param = inputs['Param']
+ grad = inputs['Grad']
+ moment1 = inputs['Moment1']
+ moment2 = inputs['Moment2']
+ lr = inputs['LearningRate']
+ beta1_pow = inputs['Beta1Pow']
+ beta2_pow = inputs['Beta2Pow']
+
+ beta1 = attributes['beta1']
+ beta2 = attributes['beta2']
+ epsilon = attributes['epsilon']
+
+ moment1_out = beta1 * moment1 + (1 - beta1) * grad
+ moment2_out = beta2 * moment2 + (1 - beta2) * np.square(grad)
+ beta1_pow_out = beta1_pow * beta1
+ beta2_pow_out = beta2_pow * beta2
+ lr_t = lr * np.sqrt(1 - beta2_pow_out) / (1 - beta1_pow_out)
+ param_out = param - lr_t * (moment1_out / (np.sqrt(moment2_out) + epsilon))
+ return param_out, moment1_out, moment2_out, beta1_pow_out, beta2_pow_out
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/python/paddle/v2/framework/tests/test_conv2d_op.py b/python/paddle/v2/framework/tests/test_conv2d_op.py
index 118a5fc1cd..2fb808944a 100644
--- a/python/paddle/v2/framework/tests/test_conv2d_op.py
+++ b/python/paddle/v2/framework/tests/test_conv2d_op.py
@@ -3,70 +3,56 @@ import numpy as np
from op_test import OpTest
+def conv2d_forward_naive(input, filter, group, conv_param):
+ in_n, in_c, in_h, in_w = input.shape
+ out_c, f_c, f_h, f_w = filter.shape
+ assert f_c * group == in_c
+ assert np.mod(out_c, group) == 0
+ sub_out_c = out_c / group
+
+ stride, pad = conv_param['stride'], conv_param['pad']
+ out_h = 1 + (in_h + 2 * pad[0] - f_h) / stride[0]
+ out_w = 1 + (in_w + 2 * pad[1] - f_w) / stride[1]
+ out = np.zeros((in_n, out_c, out_h, out_w))
+
+ input_pad = np.pad(input, ((0, ), (0, ), (pad[0], ), (pad[1], )),
+ mode='constant',
+ constant_values=0)
+ for i in range(out_h):
+ for j in range(out_w):
+ for g in range(group):
+ input_pad_masked = \
+ input_pad[:, g * f_c:(g + 1) * f_c,
+ i * stride[0]:i * stride[0] + f_h,
+ j * stride[1]:j * stride[1] + f_w]
+
+ f_sub = filter[g * sub_out_c:(g + 1) * sub_out_c, :, :, :]
+ for k in range(sub_out_c):
+ out[:, g * sub_out_c + k, i, j] = \
+ np.sum(input_pad_masked * f_sub[k, :, :, :],
+ axis=(1, 2, 3))
+
+ return out
+
+
class TestConv2dOp(OpTest):
def setUp(self):
- self.init_groups()
- self.op_type = "conv2d"
- batch_size = 2
- input_channels = 3
- input_height = 5
- input_width = 5
- output_channels = 6
- filter_height = 3
- filter_width = 3
- stride = 1
- padding = 0
- output_height = (input_height - filter_height + 2 * padding
- ) / stride + 1
- output_width = (input_width - filter_width + 2 * padding) / stride + 1
- input = np.random.random((batch_size, input_channels, input_height,
- input_width)).astype("float32")
-
- filter = np.random.random(
- (output_channels, input_channels / self.groups, filter_height,
- filter_width)).astype("float32")
- output = np.ndarray(
- (batch_size, output_channels, output_height, output_width))
+ self.init_op_type()
+ self.init_group()
+ self.init_test_case()
+
+ conv2d_param = {'stride': self.stride, 'pad': self.pad}
+ input = np.random.random(self.input_size).astype("float32")
+ filter = np.random.random(self.filter_size).astype("float32")
+ output = conv2d_forward_naive(input, filter, self.groups, conv2d_param)
self.inputs = {'Input': input, 'Filter': filter}
self.attrs = {
- 'strides': [1, 1],
- 'paddings': [0, 0],
- 'groups': self.groups
+ 'strides': self.stride,
+ 'paddings': self.pad,
+ 'groups': self.groups,
+ 'dilations': self.dilations
}
-
- output_group_channels = output_channels / self.groups
- input_group_channels = input_channels / self.groups
- for batchid in xrange(batch_size):
- for group in xrange(self.groups):
- for outchannelid in range(group * output_group_channels,
- (group + 1) * output_group_channels):
- for rowid in xrange(output_height):
- for colid in xrange(output_width):
- start_h = (rowid * stride) - padding
- start_w = (colid * stride) - padding
- output_value = 0.0
- for inchannelid in range(
- group * input_group_channels,
- (group + 1) * input_group_channels):
- for frowid in xrange(filter_height):
- for fcolid in xrange(filter_width):
- input_value = 0.0
- inrowid = start_h + frowid
- incolid = start_w + fcolid
- if ((inrowid >= 0 and
- inrowid < input_height) and
- (incolid >= 0 and
- incolid < input_width)):
- input_value = input[batchid][
- inchannelid][inrowid][incolid]
- filter_value = filter[outchannelid][
- inchannelid % input_group_channels][
- frowid][fcolid]
- output_value += input_value * filter_value
- output[batchid][outchannelid][rowid][
- colid] = output_value
-
self.outputs = {'Output': output}
def test_check_output(self):
@@ -90,14 +76,47 @@ class TestConv2dOp(OpTest):
max_relative_error=0.05,
no_grad_set=set(['Input']))
- def init_groups(self):
+ def init_test_case(self):
+ # self.groups = 1
+ # self.op_type = "conv2d"
+ self.pad = [0, 0]
+ self.stride = [1, 1]
+ self.dilations = [1, 1]
+ self.input_size = [2, 3, 5, 5] # NCHW
+ assert np.mod(self.input_size[1], self.groups) == 0
+ f_c = self.input_size[1] / self.groups
+ self.filter_size = [6, f_c, 3, 3]
+
+ def init_group(self):
self.groups = 1
+ def init_op_type(self):
+ self.op_type = "conv2d"
+
class TestWithGroup(TestConv2dOp):
- def init_groups(self):
+ def init_group(self):
self.groups = 3
+ def init_op_type(self):
+ self.op_type = "conv2d"
+
+
+class TestCudnn(TestConv2dOp):
+ def init_group(self):
+ self.groups = 1
+
+ def init_op_type(self):
+ self.op_type = "conv_cudnn"
+
+
+class TestCudnnWithGroup(TestConv2dOp):
+ def init_group(self):
+ self.groups = 3
+
+ def init_op_type(self):
+ self.op_type = "conv_cudnn"
+
if __name__ == '__main__':
unittest.main()
diff --git a/python/paddle/v2/framework/tests/test_decayed_adagrad_op.py b/python/paddle/v2/framework/tests/test_decayed_adagrad_op.py
new file mode 100644
index 0000000000..674c3fda5c
--- /dev/null
+++ b/python/paddle/v2/framework/tests/test_decayed_adagrad_op.py
@@ -0,0 +1,71 @@
+import unittest
+import numpy as np
+from op_test import OpTest
+
+
+class TestDecayedAdagradOp1(OpTest):
+ ''' Test DecayedAdagrad operator with explicit attributes
+ '''
+
+ def setUp(self):
+ self.op_type = "decayed_adagrad"
+
+ param = np.random.random((123, 321)).astype("float32")
+ grad = np.random.random((123, 321)).astype("float32")
+ moment = np.zeros((123, 321)).astype("float32")
+ lr = 0.01
+ decay = 0.80
+ epsilon = 1e-8
+
+ self.inputs = {
+ 'Param': param,
+ 'Grad': grad,
+ 'Moment': moment,
+ 'LearningRate': np.array([lr]).astype("float32")
+ }
+
+ self.attrs = {'decay': decay, 'epsilon': epsilon}
+
+ moment_out = decay * moment + (1 - decay) * grad * grad
+ param_out = param - lr * grad / (np.sqrt(moment_out) + epsilon)
+
+ self.outputs = {'ParamOut': param_out, 'MomentOut': moment_out}
+
+ def test_check_output(self):
+ self.check_output()
+
+
+class TestDecayedAdagradOp2(OpTest):
+ ''' Test DecayedAdagrad operator with default attributes
+ '''
+
+ def setUp(self):
+ self.op_type = "decayed_adagrad"
+
+ param = np.random.random((123, 321)).astype("float32")
+ grad = np.random.random((123, 321)).astype("float32")
+ moment = np.zeros((123, 321)).astype("float32")
+ lr = 0.01
+ decay = 0.95
+ epsilon = 1e-6
+
+ self.inputs = {
+ 'Param': param,
+ 'Grad': grad,
+ 'Moment': moment,
+ 'LearningRate': np.array([lr]).astype("float32")
+ }
+
+ self.attrs = {'decay': decay, 'epsilon': epsilon}
+
+ moment_out = decay * moment + (1 - decay) * grad * grad
+ param_out = param - lr * grad / (np.sqrt(moment_out) + epsilon)
+
+ self.outputs = {'ParamOut': param_out, 'MomentOut': moment_out}
+
+ def test_check_output(self):
+ self.check_output()
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/python/paddle/v2/framework/tests/test_dynamic_recurrent_op.py b/python/paddle/v2/framework/tests/test_dynamic_recurrent_op.py
new file mode 100644
index 0000000000..b4629a3adb
--- /dev/null
+++ b/python/paddle/v2/framework/tests/test_dynamic_recurrent_op.py
@@ -0,0 +1,111 @@
+import logging
+import paddle.v2.framework.core as core
+import unittest
+from paddle.v2.framework.op import Operator, DynamicRecurrentOp
+import numpy as np
+
+
+def create_tensor(scope, name, shape, np_data):
+ tensor = scope.new_var(name).get_tensor()
+ tensor.set_dims(shape)
+ tensor.set(np_data, core.CPUPlace())
+ return tensor
+
+
+class DynamicRecurrentOpTest(unittest.TestCase):
+ '''
+ Test RNNOp
+
+ equation:
+ h_t = \sigma (W x_t + U h_{t-1})
+ weights:
+ - W
+ - U
+ vars:
+ - x
+ memories:
+ - h
+ outputs:
+ - h
+ '''
+
+ # for siplicity, just one level LoD
+ lod_py = [[0, 4, 7, 9, 10]]
+ input_dim = 30
+ num_sents = len(lod_py[0]) - 1
+ weight_dim = 15
+
+ def forward(self):
+ self.scope = core.Scope()
+ self.create_global_variables()
+ self.create_rnn_op()
+ self.create_step_net()
+ ctx = core.DeviceContext.create(core.CPUPlace())
+ self.rnnop.run(self.scope, ctx)
+ state = self.rnnop.get_state("h@mem")
+ print 'state size: ', state.size()
+
+ step_inputs = self.rnnop.get_step_input("x")
+ print "x size ", step_inputs.size()
+ for i in range(step_inputs.size()):
+ print "x %d" % i, np.array(step_inputs.read(i).get_dims())
+ step_outputs = self.rnnop.get_step_output('h@mem')
+ print 'step_outputs.size ', step_outputs.size()
+ output = self.scope.find_var("h@mem").get_tensor()
+
+ print 'output', np.array(output).shape
+
+ def create_global_variables(self):
+ x = np.random.normal(size=(self.lod_py[0][-1],
+ self.input_dim)).astype("float32")
+ W = np.random.normal(size=(self.input_dim,
+ self.input_dim)).astype("float32")
+ U = np.random.normal(size=(self.input_dim,
+ self.input_dim)).astype("float32")
+ h_boot = np.random.normal(size=(self.num_sents,
+ self.input_dim)).astype("float32")
+ # create inlink
+ x_tensor = create_tensor(self.scope, "x",
+ [self.num_sents, self.input_dim], x)
+ x_tensor.set_lod(self.lod_py)
+ create_tensor(self.scope, "W", [self.input_dim, self.input_dim], W)
+ create_tensor(self.scope, "U", [self.input_dim, self.input_dim], U)
+ create_tensor(self.scope, "h_boot", [self.num_sents, self.input_dim],
+ h_boot)
+ self.scope.new_var("step_scopes")
+ self.scope.new_var("h@mem")
+
+ def create_rnn_op(self):
+ # create RNNOp
+ self.rnnop = DynamicRecurrentOp(
+ # inputs
+ inlinks=["x"],
+ boot_memories=["h_boot"],
+ step_net="stepnet",
+ # outputs
+ outlinks=["h@mem"],
+ step_scopes="step_scopes",
+ # attributes
+ pre_memories=["h@pre"],
+ memories=["h@mem"])
+
+ def create_step_net(self):
+ stepnet = core.Net.create()
+ x_fc_op = Operator("mul", X="x", Y="W", Out="Wx")
+ h_fc_op = Operator("mul", X="h@pre", Y="U", Out="Uh")
+ sum_op = Operator("sum", X=["Wx", "Uh"], Out="sum")
+ sig_op = Operator("sigmoid", X="sum", Y="h@mem")
+
+ for op in [x_fc_op, h_fc_op, sum_op, sig_op]:
+ stepnet.append_op(op)
+ stepnet.complete_add_op(True)
+ self.rnnop.set_stepnet(stepnet)
+
+ def test_forward(self):
+ print 'test recurrent op forward'
+ pd_output = self.forward()
+ print 'pd_output', pd_output
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/python/paddle/v2/framework/tests/test_margin_rank_loss_op.py b/python/paddle/v2/framework/tests/test_margin_rank_loss_op.py
new file mode 100644
index 0000000000..63378cbc4e
--- /dev/null
+++ b/python/paddle/v2/framework/tests/test_margin_rank_loss_op.py
@@ -0,0 +1,39 @@
+import unittest
+import numpy as np
+from op_test import OpTest
+
+
+class TestMarginRankLossOp(OpTest):
+ def setUp(self):
+ self.op_type = "margin_rank_loss"
+ batch_size = 5
+ margin = 0.5
+ # labels_{i} = {-1, 1}
+ label = 2 * np.random.randint(
+ 0, 2, size=(batch_size, 1)).astype("float32") - 1
+ x1 = np.random.random((batch_size, 1)).astype("float32")
+ x2 = np.random.random((batch_size, 1)).astype("float32")
+ # loss = max(0, -label * (x1 - x2) + margin)
+ loss = -label * (x1 - x2) + margin
+ loss = np.where(loss > 0, loss, 0)
+ act = np.where(loss > 0, 1., 0.)
+
+ self.attrs = {'margin': margin}
+ self.inputs = {'Label': label, 'X1': x1, 'X2': x2}
+ self.outputs = {'Activated': act, 'Out': loss}
+
+ def test_check_output(self):
+ self.check_output()
+
+ def test_check_grad(self):
+ self.check_grad(["X1", "X2"], "Out")
+
+ def test_check_grad_ignore_x1(self):
+ self.check_grad(["X2"], "Out", no_grad_set=set('X1'))
+
+ def test_check_grad_ignore_x2(self):
+ self.check_grad(["X1"], "Out", no_grad_set=set('X2'))
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/python/paddle/v2/framework/tests/test_operator_desc.py b/python/paddle/v2/framework/tests/test_operator_desc.py
new file mode 100644
index 0000000000..d7a85d8e4e
--- /dev/null
+++ b/python/paddle/v2/framework/tests/test_operator_desc.py
@@ -0,0 +1,76 @@
+import unittest
+from paddle.v2.framework.framework import Variable, g_program
+import paddle.v2.framework.core as core
+
+
+class TestOperator(unittest.TestCase):
+ def test_error_type(self):
+ block = g_program.create_block()
+ try:
+ block.append_op()
+ self.assertFail()
+ except ValueError as v_err:
+ self.assertEqual(
+ v_err.message,
+ "`type` to initilized an Operator can not be None.")
+ try:
+ block.append_op(type="no_such_op")
+ self.assertFail()
+ except AssertionError as a_err:
+ self.assertEqual(a_err.message,
+ "Operator \"no_such_op\" has not been registered.")
+
+ def test_op_desc_creation(self):
+ block = g_program.current_block()
+ mul_x = block.create_var(
+ dtype="float32", shape=[5, 10], lod_level=0, name="mul.x")
+ mul_y = block.create_var(
+ dtype="float32", shape=[10, 8], lod_level=0, name="mul.y")
+ mul_out = block.create_var(
+ dtype="float32", shape=[5, 8], lod_level=0, name="mul.out")
+ mul_op = block.append_op(
+ type="mul",
+ inputs={"X": [mul_x],
+ "Y": mul_y},
+ outputs={"Out": [mul_out]},
+ attrs={"x_num_col_dims": 1})
+ self.assertEqual(mul_op.type, "mul")
+ self.assertEqual(mul_op.input_names, ["X", "Y"])
+ self.assertEqual(mul_op.input("X"), ["mul.x"])
+ self.assertEqual(mul_op.input("Y"), ["mul.y"])
+ self.assertEqual(mul_op.output_names, ["Out"])
+ self.assertEqual(mul_op.output("Out"), ["mul.out"])
+ self.assertEqual(
+ set(mul_op.attr_names), set(["x_num_col_dims", "y_num_col_dims"]))
+ self.assertEqual(mul_op.has_attr("x_num_col_dims"), True)
+ self.assertEqual(mul_op.attr_type("x_num_col_dims"), core.AttrType.INT)
+ self.assertEqual(mul_op.attr("x_num_col_dims"), 1)
+ self.assertEqual(mul_op.has_attr("y_num_col_dims"), True)
+ self.assertEqual(mul_op.attr_type("y_num_col_dims"), core.AttrType.INT)
+ self.assertEqual(mul_op.attr("y_num_col_dims"), 1)
+ self.assertEqual(mul_out.op, mul_op)
+
+ def test_mult_input(self):
+ block = g_program.current_block()
+ sum_x1 = block.create_var(
+ dtype="int", shape=[3, 4], lod_level=0, name="sum.x1")
+ sum_x2 = block.create_var(
+ dtype="int", shape=[3, 4], lod_level=0, name="sum.x2")
+ sum_x3 = block.create_var(
+ dtype="int", shape=[3, 4], lod_level=0, name="sum.x3")
+ sum_out = block.create_var(
+ dtype="int", shape=[3, 4], lod_level=0, name="sum.out")
+ sum_op = block.append_op(
+ type="sum",
+ inputs={"X": [sum_x1, sum_x2, sum_x3]},
+ outputs={"Out": sum_out})
+ self.assertEqual(sum_op.type, "sum")
+ self.assertEqual(sum_op.input_names, ["X"])
+ self.assertEqual(sum_op.input("X"), ["sum.x1", "sum.x2", "sum.x3"])
+ self.assertEqual(sum_op.output_names, ["Out"])
+ self.assertEqual(sum_op.output("Out"), ["sum.out"])
+ self.assertEqual(sum_out.op, sum_op)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/python/paddle/v2/framework/tests/test_parameter.py b/python/paddle/v2/framework/tests/test_parameter.py
index 3b5d38f257..1ac0cdd99f 100644
--- a/python/paddle/v2/framework/tests/test_parameter.py
+++ b/python/paddle/v2/framework/tests/test_parameter.py
@@ -1,5 +1,5 @@
import unittest
-from paddle.v2.framework.graph import g_program
+from paddle.v2.framework.framework import g_program
import paddle.v2.framework.core as core
diff --git a/python/paddle/v2/framework/tests/test_program.py b/python/paddle/v2/framework/tests/test_program.py
index b82d1760d6..64b781e6ea 100644
--- a/python/paddle/v2/framework/tests/test_program.py
+++ b/python/paddle/v2/framework/tests/test_program.py
@@ -1,5 +1,7 @@
import unittest
-from paddle.v2.framework.graph import g_program
+
+import paddle.v2.framework.core as core
+from paddle.v2.framework.framework import g_program
class TestProgram(unittest.TestCase):
@@ -31,6 +33,34 @@ class TestProgram(unittest.TestCase):
self.assertEqual(1, b.idx)
self.assertEqual(0, b.parent_idx)
+ def test_append_backward(self):
+ prog = core.ProgramDesc.__create_program_desc__()
+ self.assertIsNotNone(prog)
+ block = prog.block(0)
+ self.assertIsNotNone(block)
+
+ mul_op_desc = block.append_op()
+ mul_op_desc.set_type("mul")
+ mul_op_desc.set_input("X", ["x1"])
+ mul_op_desc.set_input("Y", ["y1"])
+ mul_op_desc.set_output("Out", ["out1"])
+
+ sum_op_desc = block.append_op()
+ sum_op_desc.set_type("elementwise_add")
+ sum_op_desc.set_input("X", ["out1"])
+ sum_op_desc.set_input("Y", ["b1"])
+ sum_op_desc.set_output("Out", ["out2"])
+
+ expect_ops = [
+ "mul", "elementwise_add", "elementwise_add_grad", "mul_grad"
+ ]
+ actual_ops = []
+ prog.append_backward(set())
+ for op in block.all_ops():
+ actual_ops.append(op.type())
+ print(actual_ops)
+ self.assertEqual(actual_ops, expect_ops)
+
if __name__ == '__main__':
unittest.main()
diff --git a/python/paddle/v2/framework/tests/test_protobuf_descs.py b/python/paddle/v2/framework/tests/test_protobuf_descs.py
index 2b7ba6688a..af5ed6801f 100644
--- a/python/paddle/v2/framework/tests/test_protobuf_descs.py
+++ b/python/paddle/v2/framework/tests/test_protobuf_descs.py
@@ -53,7 +53,13 @@ class TestOpDesc(unittest.TestCase):
self.assertEqual(8, len(op.attr_names()))
op.set_block_attr("block_attr", prog.block(0))
- self.assertEqual(0, op.get_block_attr("block_attr"))
+ self.assertEqual(0, op.block_attr("block_attr"))
+
+ mul_op = block.append_op()
+ mul_op.set_type("mul")
+ mul_op.check_attrs()
+ self.assertEqual(mul_op.attr("x_num_col_dims"), 1)
+ self.assertEqual(mul_op.attr("y_num_col_dims"), 1)
class TestProgramDesc(unittest.TestCase):
diff --git a/python/paddle/v2/framework/tests/test_seq_concat_op.py b/python/paddle/v2/framework/tests/test_seq_concat_op.py
new file mode 100644
index 0000000000..abd2ebf0b2
--- /dev/null
+++ b/python/paddle/v2/framework/tests/test_seq_concat_op.py
@@ -0,0 +1,79 @@
+import unittest
+import numpy as np
+import sys
+from op_test import OpTest
+
+
+class TestConcatOp(OpTest):
+ def set_data(self):
+ # two level, batch size is 3
+ x0 = np.random.random((4, 6, 3)).astype('float32')
+ lod0 = [[0, 2, 4], [0, 1, 2, 3, 4]]
+ x1 = np.random.random((4, 8, 3)).astype('float32')
+ lod1 = [[0, 2, 4], [0, 1, 2, 3, 4]]
+ axis = 1
+ level = 1
+ self.inputs = {'X': [('x0', (x0, lod0)), ('x1', (x1, lod1))]}
+ self.attrs = {'axis': axis, 'level': level}
+ outs = []
+ for i in range(4):
+ sub_x0 = x0[lod0[level][i]:lod0[level][i + 1], :]
+ sub_x1 = x1[lod1[level][i]:lod1[level][i + 1], :]
+ outs.append(np.concatenate((sub_x0, sub_x1), axis=axis))
+
+ self.outputs = {'Out': np.concatenate(outs, axis=0)}
+
+ def setUp(self):
+ self.op_type = "sequence_concat"
+ self.set_data()
+
+ def test_check_output(self):
+ self.check_output()
+
+ def test_check_grad(self):
+ self.check_grad(['x0'], 'Out')
+
+
+class TestConcatOpDiffLod(TestConcatOp):
+ def set_data(self):
+ # two level, batch size is 3
+ x0 = np.random.random((4, 6, 3)).astype('float32')
+ lod0 = [[0, 2, 4], [0, 1, 2, 3, 4]]
+ x1 = np.random.random((5, 6, 3)).astype('float32')
+ lod1 = [[0, 3, 5], [0, 1, 2, 3, 5]]
+ axis = 0
+ level = 1
+ self.inputs = {'X': [('x0', (x0, lod0)), ('x1', (x1, lod1))]}
+ self.attrs = {'axis': axis, 'level': level}
+ outs = []
+ for i in range(4):
+ sub_x0 = x0[lod0[level][i]:lod0[level][i + 1], :]
+ sub_x1 = x1[lod1[level][i]:lod1[level][i + 1], :]
+ outs.append(np.concatenate((sub_x0, sub_x1), axis=axis))
+
+ self.outputs = {'Out': np.concatenate(outs, axis=0)}
+
+
+class TestConcatOpLevelZero(TestConcatOp):
+ def set_data(self):
+ # two level, batch size is 3
+ x0 = np.random.random((4, 3, 4)).astype('float32')
+ lod0 = [[0, 2, 4], [0, 1, 2, 3, 4]]
+ x1 = np.random.random((5, 3, 4)).astype('float32')
+ lod1 = [[0, 3, 5], [0, 1, 3, 4, 5]]
+ axis = 0
+ level = 0
+ self.inputs = {'X': [('x0', (x0, lod0)), ('x1', (x1, lod1))]}
+ self.attrs = {'axis': axis, 'level': level}
+ outs = []
+ for i in range(2):
+ sub_x0 = x0[lod0[level][i]:lod0[level][i + 1], :]
+ sub_x1 = x1[lod1[level][i]:lod1[level][i + 1], :]
+ outs.append(np.concatenate((sub_x0, sub_x1), axis=axis))
+
+ self.outputs = {'Out': np.concatenate(outs, axis=0)}
+
+
+if __name__ == '__main__':
+ sys.exit(0)
+ unittest.main()
diff --git a/python/paddle/v2/framework/tests/test_variable.py b/python/paddle/v2/framework/tests/test_variable.py
index 8ea1083ff6..695aaaee6c 100644
--- a/python/paddle/v2/framework/tests/test_variable.py
+++ b/python/paddle/v2/framework/tests/test_variable.py
@@ -1,5 +1,5 @@
import unittest
-from paddle.v2.framework.graph import Variable, g_program
+from paddle.v2.framework.framework import Variable, g_program
import paddle.v2.framework.core as core
import numpy as np