From 53b0e427092219b402f0ed6fab4235c3b70fdc7c Mon Sep 17 00:00:00 2001 From: hedaoyuan Date: Thu, 17 Aug 2017 16:19:59 +0800 Subject: [PATCH 01/13] Add EigenGemm. --- paddle/function/EigenGemm.cpp | 92 ++++++++++++++++++++++++++++++ paddle/function/GemmFunctor.cpp | 85 ++++++++++++++++++++++++++++ paddle/function/GemmFunctor.h | 99 +++++++++++---------------------- 3 files changed, 211 insertions(+), 65 deletions(-) create mode 100644 paddle/function/EigenGemm.cpp create mode 100644 paddle/function/GemmFunctor.cpp diff --git a/paddle/function/EigenGemm.cpp b/paddle/function/EigenGemm.cpp new file mode 100644 index 0000000000..0b4220fcbe --- /dev/null +++ b/paddle/function/EigenGemm.cpp @@ -0,0 +1,92 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include +#include "unsupported/Eigen/CXX11/Tensor" + +namespace paddle { + +template +struct EigenBlasGemm { + typedef Eigen::TensorMap, + Eigen::Aligned> + Matrix; + + static void compute(const bool transA, + const bool transB, + const int M, + const int N, + const int K, + const T alpha, + const T* A, + const int lda, + const T* B, + const int ldb, + const T beta, + T* C, + const int ldc) { + Eigen::array sizeA; + if (transA) { + sizeA[0] = K; + sizeA[1] = M; + CHECK_EQ(M, lda); + } else { + sizeA[0] = M; + sizeA[1] = K; + CHECK_EQ(K, lda); + } + Eigen::array sizeB; + if (transB) { + sizeB[0] = N; + sizeB[1] = K; + CHECK_EQ(K, ldb); + } else { + sizeB[0] = K; + sizeB[1] = N; + CHECK_EQ(N, ldb); + } + Eigen::array sizeC; + sizeC[0] = M; + sizeC[1] = N; + CHECK_EQ(N, ldc); + + const Matrix a(const_cast(A), sizeA); + const Matrix b(const_cast(B), sizeB); + Matrix c(C, sizeC); + + typedef typename Eigen::Tensor::DimensionPair DimPair; + Eigen::array dims; + dims[0] = DimPair(1, 0); + dims[0].first = transA ? 0 : 1; + dims[0].second = transB ? 1 : 0; + + Eigen::DefaultDevice device; + if (alpha == T(1) && beta == T(0)) { + c.device(device) = a.contract(b, dims); + } else if (alpha == T(1) && beta == T(1)) { + c.device(device) += a.contract(b, dims); + } else { + c.device(device) = + c.constant(alpha) * a.contract(b, dims) + c.constant(beta) * c; + } + } +}; + +#ifdef PADDLE_TYPE_DOUBLE +template class EigenBlasGemm; +#else +template class EigenBlasGemm; +#endif + +} // namespace paddle diff --git a/paddle/function/GemmFunctor.cpp b/paddle/function/GemmFunctor.cpp new file mode 100644 index 0000000000..8df9b884fe --- /dev/null +++ b/paddle/function/GemmFunctor.cpp @@ -0,0 +1,85 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "GemmFunctor.h" +#include "paddle/math/MathFunctions.h" + +namespace paddle { + +template +struct BlasGemm { + static void compute(const bool transA, + const bool transB, + const int M, + const int N, + const int K, + const T alpha, + const T* A, + const int lda, + const T* B, + const int ldb, + const T beta, + T* C, + const int ldc) { + gemm(transA == false ? CblasNoTrans : CblasTrans, + transB == false ? CblasNoTrans : CblasTrans, + M, + N, + K, + alpha, + A, + lda, + B, + ldb, + beta, + C, + ldc); + } +}; + +template +struct BlasGemm { + static void compute(const bool transA, + const bool transB, + const int M, + const int N, + const int K, + const T alpha, + const T* A, + const int lda, + const T* B, + const int ldb, + const T beta, + T* C, + const int ldc) { + hl_matrix_mul((T*)A, + transA == false ? HPPL_OP_N : HPPL_OP_T, + (T*)B, + transB == false ? HPPL_OP_N : HPPL_OP_T, + C, + M, + N, + K, + alpha, + beta, + lda, + ldb, + ldc); + } +}; + +template class BlasGemm; +template class BlasGemm; + +} // namespace paddle diff --git a/paddle/function/GemmFunctor.h b/paddle/function/GemmFunctor.h index d5db5cf5e7..0809953b4e 100644 --- a/paddle/function/GemmFunctor.h +++ b/paddle/function/GemmFunctor.h @@ -14,7 +14,7 @@ limitations under the License. */ #pragma once -#include "paddle/math/MathFunctions.h" +#include "TensorType.h" namespace paddle { @@ -24,73 +24,42 @@ namespace paddle { // of MatMulFunction, we need to consider the reconstruction of hl_matrix_mul // interface. template -class GemmFunctor { -public: - void operator()(const CBLAS_TRANSPOSE transA, - const CBLAS_TRANSPOSE TransB, - const int M, - const int N, - const int K, - const T alpha, - const T* A, - const int lda, - const T* B, - const int ldb, - const T beta, - T* C, - const int ldc); +struct BlasGemm { + static void compute(const bool transA, + const bool transB, + const int M, + const int N, + const int K, + const T alpha, + const T* A, + const int lda, + const T* B, + const int ldb, + const T beta, + T* C, + const int ldc); }; +// TODO(hedaoyuan): Since the definition of the real type in the Paddle +// conflicts with the Eigen library, so compile the Eigen code can not +// include the Paddle header file. And need an EigenBlasGemm template class +// that does not contain the DeviceType parameter. +// I will fix this problem and merge BlasGemm and EigenBlasGemm into one. template -class GemmFunctor { -public: - void operator()(const CBLAS_TRANSPOSE transA, - const CBLAS_TRANSPOSE TransB, - const int M, - const int N, - const int K, - const T alpha, - const T* A, - const int lda, - const T* B, - const int ldb, - const T beta, - T* C, - const int ldc) { - gemm(transA, TransB, M, N, K, alpha, A, lda, B, ldb, beta, C, ldc); - } -}; - -template -class GemmFunctor { -public: - void operator()(const CBLAS_TRANSPOSE transA, - const CBLAS_TRANSPOSE TransB, - const int M, - const int N, - const int K, - const T alpha, - const T* A, - const int lda, - const T* B, - const int ldb, - const T beta, - T* C, - const int ldc) { - hl_matrix_mul((T*)A, - transA == CblasNoTrans ? HPPL_OP_N : HPPL_OP_T, - (T*)B, - TransB == CblasNoTrans ? HPPL_OP_N : HPPL_OP_T, - C, - M, - N, - K, - alpha, - beta, - lda, - ldb, - ldc); - } +struct EigenBlasGemm { + static void compute(const bool transA, + const bool transB, + const int M, + const int N, + const int K, + const T alpha, + const T* A, + const int lda, + const T* B, + const int ldb, + const T beta, + T* C, + const int ldc); }; } // namespace paddle From ec2ba242060fc10b2045533fdcb410cfbd473cec Mon Sep 17 00:00:00 2001 From: hedaoyuan Date: Thu, 17 Aug 2017 16:22:30 +0800 Subject: [PATCH 02/13] Fix GemmConvFunction. --- paddle/function/CMakeLists.txt | 2 + paddle/function/GemmConvOp.cpp | 82 ++++++++++++++++------------------ 2 files changed, 41 insertions(+), 43 deletions(-) diff --git a/paddle/function/CMakeLists.txt b/paddle/function/CMakeLists.txt index 7dfb6f61c5..9187294a49 100644 --- a/paddle/function/CMakeLists.txt +++ b/paddle/function/CMakeLists.txt @@ -4,6 +4,8 @@ file(GLOB cpp_files . *Op.cpp) list(APPEND h_files Function.h) list(APPEND cpp_files Function.cpp) list(APPEND cpp_files BufferArg.cpp) +list(APPEND cpp_files GemmFunctor.cpp) +list(APPEND cpp_files EigenGemm.cpp) if(WITH_GPU) file(GLOB cu_files . *OpGpu.cu) diff --git a/paddle/function/GemmConvOp.cpp b/paddle/function/GemmConvOp.cpp index 0ada4d70a0..f8cf4ebea8 100644 --- a/paddle/function/GemmConvOp.cpp +++ b/paddle/function/GemmConvOp.cpp @@ -85,7 +85,6 @@ public: } Im2ColFunctor im2col; - GemmFunctor gemm; size_t inputOffset = imShape.getElements(); size_t outputOffset = (outputChannels / groups_) * outputHeight * outputWidth; @@ -108,19 +107,19 @@ public: int M = outputChannels / groups_; int N = outputHeight * outputWidth; int K = inputChannels / groups_ * filterHeight * filterWidth; - gemm(CblasNoTrans, - CblasNoTrans, - M, - N, - K, - 1.0f, - filterData + g * filterOffset, - K, - colData, - N, - beta, - outputData + g * outputOffset, - N); + BlasGemm::compute(false, + false, + M, + N, + K, + 1.0f, + filterData + g * filterOffset, + K, + colData, + N, + beta, + outputData + g * outputOffset, + N); } inputData += inputChannels * inputHeight * inputWidth; outputData += outputChannels * outputHeight * outputWidth; @@ -188,8 +187,6 @@ public: } Col2ImFunctor col2im; - GemmFunctor gemm; - size_t inputOffset = imShape.getElements(); size_t outputOffset = (outputChannels / groups_) * outputHeight * outputWidth; @@ -205,19 +202,19 @@ public: colData = inputGrad + g * inputOffset; scale = 1.0f; } - gemm(CblasTrans, - CblasNoTrans, - M, - N, - K, - 1.0f, - filterData + g * filterOffset, - M, - outputGrad + g * outputOffset, - N, - scale, - colData, - N); + BlasGemm::compute(true, + false, + M, + N, + K, + 1.0f, + filterData + g * filterOffset, + M, + outputGrad + g * outputOffset, + N, + scale, + colData, + N); if (needIm2col) { col2im(inputGrad + g * inputOffset, imShape, @@ -299,7 +296,6 @@ public: } Im2ColFunctor im2col; - GemmFunctor gemm; size_t inputOffset = imShape.getElements(); size_t outputOffset = (outputChannels / groups_) * outputHeight * outputWidth; @@ -321,19 +317,19 @@ public: int M = outputChannels / groups_; int K = outputHeight * outputWidth; int N = inputChannels / groups_ * filterHeight * filterWidth; - gemm(CblasNoTrans, - CblasTrans, - M, - N, - K, - 1.0f, - outputGrad + g * outputOffset, - K, - colData, - K, - i == 0 ? beta : 1.0f, - filterGrad + g * filterOffset, - N); + BlasGemm::compute(false, + true, + M, + N, + K, + 1.0f, + outputGrad + g * outputOffset, + K, + colData, + K, + i == 0 ? beta : 1.0f, + filterGrad + g * filterOffset, + N); } inputData += inputChannels * inputHeight * inputWidth; outputGrad += outputChannels * outputHeight * outputWidth; From adcca2cc064182cd75809dd1e3d8c64329a0b0de Mon Sep 17 00:00:00 2001 From: hedaoyuan Date: Thu, 17 Aug 2017 16:40:38 +0800 Subject: [PATCH 03/13] Add PADDLE_USE_EIGEN_FOR_BLAS macro. --- CMakeLists.txt | 1 + cmake/configure.cmake | 4 ++++ paddle/function/GemmFunctor.cpp | 5 +++++ 3 files changed, 10 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index dcd1218a5b..28bbfd7916 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -55,6 +55,7 @@ option(WITH_C_API "Compile PaddlePaddle with C-API(Prediction)" OFF) option(WITH_GOLANG "Compile PaddlePaddle with GOLANG" OFF) option(GLIDE_INSTALL "Download and install go dependencies " ON) option(USE_NNPACK "Compile PaddlePaddle with NNPACK library" OFF) +option(USE_EIGEN_FOR_BLAS "Use matrix multiplication in Eigen" OFF) # CMAKE_BUILD_TYPE if(NOT CMAKE_BUILD_TYPE) diff --git a/cmake/configure.cmake b/cmake/configure.cmake index 209f9078a6..51c3b918cc 100644 --- a/cmake/configure.cmake +++ b/cmake/configure.cmake @@ -28,6 +28,10 @@ if(NOT WITH_TIMER) add_definitions(-DPADDLE_DISABLE_TIMER) endif(NOT WITH_TIMER) +if(USE_EIGEN_FOR_BLAS) + add_definitions(-DPADDLE_USE_EIGEN_FOR_BLAS) +endif(USE_EIGEN_FOR_BLAS) + if(NOT WITH_PROFILER) add_definitions(-DPADDLE_DISABLE_PROFILER) endif(NOT WITH_PROFILER) diff --git a/paddle/function/GemmFunctor.cpp b/paddle/function/GemmFunctor.cpp index 8df9b884fe..dc83278d8e 100644 --- a/paddle/function/GemmFunctor.cpp +++ b/paddle/function/GemmFunctor.cpp @@ -32,6 +32,10 @@ struct BlasGemm { const T beta, T* C, const int ldc) { +#ifdef PADDLE_USE_EIGEN_FOR_BLAS + EigenBlasGemm::compute( + transA, transB, M, N, K, alpha, A, lda, B, ldb, beta, C, ldc); +#else gemm(transA == false ? CblasNoTrans : CblasTrans, transB == false ? CblasNoTrans : CblasTrans, M, @@ -45,6 +49,7 @@ struct BlasGemm { beta, C, ldc); +#endif } }; From 6ba04dcd112e0caac46a7a829182ce00f301752f Mon Sep 17 00:00:00 2001 From: hedaoyuan Date: Thu, 17 Aug 2017 16:56:46 +0800 Subject: [PATCH 04/13] Remove the header files that do not need to be included. --- paddle/function/DepthwiseConvOp.cpp | 1 - paddle/function/DepthwiseConvOpGpu.cu | 1 - 2 files changed, 2 deletions(-) diff --git a/paddle/function/DepthwiseConvOp.cpp b/paddle/function/DepthwiseConvOp.cpp index 490e8d546c..2f3112fe65 100644 --- a/paddle/function/DepthwiseConvOp.cpp +++ b/paddle/function/DepthwiseConvOp.cpp @@ -14,7 +14,6 @@ limitations under the License. */ #include "DepthwiseConvOp.h" #include "ConvOp.h" -#include "GemmFunctor.h" namespace paddle { diff --git a/paddle/function/DepthwiseConvOpGpu.cu b/paddle/function/DepthwiseConvOpGpu.cu index 33463805cb..2d722dfcfc 100644 --- a/paddle/function/DepthwiseConvOpGpu.cu +++ b/paddle/function/DepthwiseConvOpGpu.cu @@ -13,7 +13,6 @@ See the License for the specific language governing permissions and limitations under the License. */ #include "DepthwiseConvOp.h" -#include "GemmFunctor.h" #include "paddle/math/BaseMatrix.h" namespace paddle { From 7d2ef02a993a378921a006d3575a802e5e9c5e9d Mon Sep 17 00:00:00 2001 From: guosheng Date: Thu, 17 Aug 2017 21:18:58 +0800 Subject: [PATCH 05/13] Add ScaleShiftLayer --- doc/api/v2/config/layer.rst | 5 + paddle/gserver/layers/ScaleShiftLayer.cpp | 106 ++++++++++++++++++ paddle/gserver/tests/test_LayerGrad.cpp | 15 +++ python/paddle/trainer/config_parser.py | 14 +++ .../paddle/trainer_config_helpers/layers.py | 37 ++++++ .../tests/configs/file_list.sh | 2 +- .../protostr/test_scale_shift_layer.protostr | 72 ++++++++++++ .../tests/configs/test_scale_shift_layer.py | 11 ++ 8 files changed, 261 insertions(+), 1 deletion(-) create mode 100644 paddle/gserver/layers/ScaleShiftLayer.cpp create mode 100644 python/paddle/trainer_config_helpers/tests/configs/protostr/test_scale_shift_layer.protostr create mode 100644 python/paddle/trainer_config_helpers/tests/configs/test_scale_shift_layer.py diff --git a/doc/api/v2/config/layer.rst b/doc/api/v2/config/layer.rst index cb330ea5e1..a4a843c610 100644 --- a/doc/api/v2/config/layer.rst +++ b/doc/api/v2/config/layer.rst @@ -362,6 +362,11 @@ trans .. autoclass:: paddle.v2.layer.trans :noindex: +scale_shift +----------- +.. autoclass:: paddle.v2.layer.scale_shift + :noindex: + Sampling Layers =============== diff --git a/paddle/gserver/layers/ScaleShiftLayer.cpp b/paddle/gserver/layers/ScaleShiftLayer.cpp new file mode 100644 index 0000000000..4f5b1c6225 --- /dev/null +++ b/paddle/gserver/layers/ScaleShiftLayer.cpp @@ -0,0 +1,106 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "Layer.h" + +namespace paddle { + +/** + * A layer does scaling and shifting to the input by appling a slope and + * an intercept which are trainable to the input element-wise. + * + * \f[ + * y = wx + b + * \f] + * + * Here, w is scale and b is offset, which are scalars and trainable. + * + */ + +class ScaleShiftLayer : public Layer { +protected: + std::unique_ptr scale_; + std::unique_ptr offset_; + +public: + explicit ScaleShiftLayer(const LayerConfig& config) : Layer(config) {} + + bool init(const LayerMap& layerMap, + const ParameterMap& parameterMap) override; + + void forward(PassType passType) override; + void backward(const UpdateCallback& callback = nullptr) override; +}; + +REGISTER_LAYER(scale_shift, ScaleShiftLayer); + +bool ScaleShiftLayer::init(const LayerMap& layerMap, + const ParameterMap& parameterMap) { + Layer::init(layerMap, parameterMap); + CHECK_EQ(inputLayers_.size(), 1U); + scale_.reset(new Weight(1, 1, parameters_[0])); + if (biasParameter_.get() != NULL) { + offset_ = std::unique_ptr(new Weight(1, 1, biasParameter_)); + } + return true; +} + +void ScaleShiftLayer::forward(PassType passType) { + Layer::forward(passType); + + MatrixPtr inV = getInputValue(0); + resetOutput(inV->getHeight(), inV->getWidth()); + MatrixPtr outV = getOutputValue(); + real scaleValue = scale_->getW()->getElement(0, 0); + outV->mulScalar(*inV, scaleValue); + if (offset_) { + real offsetValue = offset_->getW()->getElement(0, 0); + outV->add(offsetValue); + } +} + +void ScaleShiftLayer::backward(const UpdateCallback& callback) { + MatrixPtr inV = getInputValue(0); + MatrixPtr inG = getInputGrad(0); + MatrixPtr outV = getOutputValue(); + MatrixPtr outG = getOutputGrad(); + + /* Calculate the parameter gradient for the current layer */ + if (scale_->getWGrad()) { + MatrixPtr rowSumMtx; + Matrix::resizeOrCreate(rowSumMtx, outG->getHeight(), 1, false, useGpu_); + // this_i = scaleDest * this_i + scaleSum * \sum_j b_{ij} * c_{ij} + rowSumMtx->sumOfProducts( + /* b= */ *inV, /* c= */ *outG, /* scaleSum= */ 1, /* scaleDest= */ 0.); + // this_i = scaleDest * this_i + scaleSum * \sum_j b_{ji} + scale_->getWGrad()->sumCols( + /* b= */ *rowSumMtx, /* scaleSum= */ 1., /* scaleDest= */ 1.); + scale_->getParameterPtr()->incUpdate(callback); + } + if (offset_ && offset_->getWGrad()) { + MatrixPtr rowSumMtx; + Matrix::resizeOrCreate(rowSumMtx, outG->getHeight(), 1, false, useGpu_); + rowSumMtx->sumRows(*outG, 1., 0.); + offset_->getWGrad()->sumCols(*rowSumMtx, 1., 1.); + offset_->getParameterPtr()->incUpdate(callback); + } + + /* Calculate the input layers error */ + if (inG) { + real scaleValue = scale_->getW()->getElement(0, 0); + inG->add(*outG, scaleValue); + } +} + +} // namespace paddle diff --git a/paddle/gserver/tests/test_LayerGrad.cpp b/paddle/gserver/tests/test_LayerGrad.cpp index 0f312b6ca5..65429ebada 100644 --- a/paddle/gserver/tests/test_LayerGrad.cpp +++ b/paddle/gserver/tests/test_LayerGrad.cpp @@ -2007,6 +2007,21 @@ TEST(Layer, RowL2NormLayer) { } } +TEST(Layer, ScaleShiftLayer) { + const size_t batchSize = 128; + const size_t size = 512; + TestConfig config; + config.layerConfig.set_type("scale_shift"); + config.layerConfig.set_size(size); + config.biasSize = 1; + config.inputDefs.push_back( + {INPUT_DATA, "input", /* dim= */ size, /* paraSize= */ 1}); + config.layerConfig.add_inputs(); + for (auto useGpu : {false, true}) { + testLayerGrad(config, "scale_shift", batchSize, false, useGpu, false); + } +} + int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); initMain(argc, argv); diff --git a/python/paddle/trainer/config_parser.py b/python/paddle/trainer/config_parser.py index da99e5bd53..8d71629faa 100644 --- a/python/paddle/trainer/config_parser.py +++ b/python/paddle/trainer/config_parser.py @@ -2232,6 +2232,20 @@ class ClipLayer(LayerBase): self.config.inputs[0].clip_conf.max = max +@config_layer('scale_shift') +class ScaleShiftLayer(LayerBase): + def __init__(self, name, inputs, bias=True, **xargs): + super(ScaleShiftLayer, self).__init__( + name, 'scale_shift', 0, inputs=inputs, **xargs) + config_assert( + len(self.inputs) == 1, + 'ScaleShiftLayer must have one and only one input.') + input_layer = self.get_input_layer(0) + self.set_layer_size(input_layer.size) + self.create_input_parameter(0, 1, [1, 1]) + self.create_bias_parameter(bias, 1) + + # key: cost type # value: cost class g_cost_map = {} diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index 1bc55c8696..4c7217024a 100755 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -133,6 +133,7 @@ __all__ = [ 'clip_layer', 'slice_projection', 'kmax_sequence_score_layer', + 'scale_shift_layer', ] @@ -230,6 +231,7 @@ class LayerType(object): CLIP_LAYER = 'clip' KMAX_SEQ_SCORE = 'kmax_seq_score' + SCALE_SHIFT_LAYER = 'scale_shift' @staticmethod def is_layer_type(type_name): @@ -6210,3 +6212,38 @@ def kmax_sequence_score_layer(input, name=None, beam_size=1): return LayerOutput( name, LayerType.KMAX_SEQ_SCORE, parents=[input], size=input.size) + + +@wrap_name_default("scale_shift") +@wrap_param_attr_default() +@wrap_bias_attr_default() +def scale_shift_layer(input, name=None, param_attr=None, bias_attr=None): + """ + A layer does scaling and shifting to the input by appling a slope and + an intercept which are trainable to the input element-wise. + .. math:: + + y = w * x + b + + .. code-block:: python + + scale_shift = scale_shift_layer(input=input_layer, bias_attr=False) + + :param name: The Layer Name. + :type name: basestring + :param input: The input layer. + :type input: LayerOutput. + :param param_attr: The parameter attribute of scaling. + :type param_attr: ParameterAttribute + :param bias_attr: The parameter attribute of shifting. + :type bias_attr: ParameterAttribute + :return: LayerOutput object. + :rtype: LayerOutput + """ + Layer( + name=name, + type=LayerType.SCALE_SHIFT_LAYER, + inputs=Input(input.name, **param_attr.attr), + bias=ParamAttr.to_bias(bias_attr)) + return LayerOutput( + name, LayerType.SCALE_SHIFT_LAYER, parents=[input], size=input.size) diff --git a/python/paddle/trainer_config_helpers/tests/configs/file_list.sh b/python/paddle/trainer_config_helpers/tests/configs/file_list.sh index a61beb871a..3860699f6f 100755 --- a/python/paddle/trainer_config_helpers/tests/configs/file_list.sh +++ b/python/paddle/trainer_config_helpers/tests/configs/file_list.sh @@ -8,6 +8,6 @@ test_spp_layer test_bilinear_interp test_maxout test_bi_grumemory math_ops test_seq_concat_reshape test_pad test_smooth_l1 test_multiplex_layer test_prelu_layer test_row_conv test_detection_output_layer test_multibox_loss_layer test_recursive_topology test_gated_unit_layer test_clip_layer test_row_l2_norm_layer -test_kmax_seq_socre_layer test_seq_select_layers) +test_kmax_seq_socre_layer test_seq_select_layers test_scale_shift_layer) export whole_configs=(test_split_datasource) diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_scale_shift_layer.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_scale_shift_layer.protostr new file mode 100644 index 0000000000..efaf20f8a7 --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_scale_shift_layer.protostr @@ -0,0 +1,72 @@ +type: "nn" +layers { + name: "data" + type: "data" + size: 100 + active_type: "" +} +layers { + name: "__scale_shift_0__" + type: "scale_shift" + size: 100 + active_type: "" + inputs { + input_layer_name: "data" + input_parameter_name: "___scale_shift_0__.w0" + } + bias_parameter_name: "___scale_shift_0__.wbias" +} +layers { + name: "__scale_shift_1__" + type: "scale_shift" + size: 100 + active_type: "" + inputs { + input_layer_name: "data" + input_parameter_name: "___scale_shift_1__.w0" + } +} +parameters { + name: "___scale_shift_0__.w0" + size: 1 + initial_mean: 0.0 + initial_std: 1.0 + dims: 1 + dims: 1 + initial_strategy: 0 + initial_smart: true +} +parameters { + name: "___scale_shift_0__.wbias" + size: 1 + initial_mean: 0.0 + initial_std: 0.0 + dims: 1 + dims: 1 + initial_strategy: 0 + initial_smart: false +} +parameters { + name: "___scale_shift_1__.w0" + size: 1 + initial_mean: 0.0 + initial_std: 1.0 + dims: 1 + dims: 1 + initial_strategy: 0 + initial_smart: true +} +input_layer_names: "data" +output_layer_names: "__scale_shift_0__" +output_layer_names: "__scale_shift_1__" +sub_models { + name: "root" + layer_names: "data" + layer_names: "__scale_shift_0__" + layer_names: "__scale_shift_1__" + input_layer_names: "data" + output_layer_names: "__scale_shift_0__" + output_layer_names: "__scale_shift_1__" + is_recurrent_layer_group: false +} + diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_scale_shift_layer.py b/python/paddle/trainer_config_helpers/tests/configs/test_scale_shift_layer.py new file mode 100644 index 0000000000..818d71f15d --- /dev/null +++ b/python/paddle/trainer_config_helpers/tests/configs/test_scale_shift_layer.py @@ -0,0 +1,11 @@ +from paddle.trainer_config_helpers import * + +settings(batch_size=1000, learning_rate=1e-5) + +data = data_layer(name='data', size=100) + +scale = scale_shift_layer(input=data) + +scale_shift = scale_shift_layer(input=data, bias_attr=False) + +outputs(scale, scale_shift) From 1eb98e2fef8f9264ed9110569748a7b42ca45eb4 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Fri, 18 Aug 2017 17:19:14 -0700 Subject: [PATCH 06/13] Set the default cuDNN installation path --- cmake/cudnn.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/cudnn.cmake b/cmake/cudnn.cmake index 69f40df516..2c84061ff5 100644 --- a/cmake/cudnn.cmake +++ b/cmake/cudnn.cmake @@ -2,7 +2,7 @@ if(NOT WITH_GPU) return() endif() -set(CUDNN_ROOT "" CACHE PATH "CUDNN ROOT") +set(CUDNN_ROOT "/usr" CACHE PATH "CUDNN ROOT") find_path(CUDNN_INCLUDE_DIR cudnn.h PATHS ${CUDNN_ROOT} ${CUDNN_ROOT}/include $ENV{CUDNN_ROOT} $ENV{CUDNN_ROOT}/include ${CUDA_TOOLKIT_INCLUDE} From 430e0e418bb34d6a14662a29a3e6d5fb906c9610 Mon Sep 17 00:00:00 2001 From: hedaoyuan Date: Mon, 21 Aug 2017 10:12:25 +0800 Subject: [PATCH 07/13] Follow comments. --- paddle/function/CMakeLists.txt | 4 +++- paddle/function/EigenGemm.cpp | 3 +-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/paddle/function/CMakeLists.txt b/paddle/function/CMakeLists.txt index 9187294a49..c572a9d433 100644 --- a/paddle/function/CMakeLists.txt +++ b/paddle/function/CMakeLists.txt @@ -5,7 +5,9 @@ list(APPEND h_files Function.h) list(APPEND cpp_files Function.cpp) list(APPEND cpp_files BufferArg.cpp) list(APPEND cpp_files GemmFunctor.cpp) -list(APPEND cpp_files EigenGemm.cpp) +if(USE_EIGEN_FOR_BLAS) + list(APPEND cpp_files EigenGemm.cpp) +endif(USE_EIGEN_FOR_BLAS) if(WITH_GPU) file(GLOB cu_files . *OpGpu.cu) diff --git a/paddle/function/EigenGemm.cpp b/paddle/function/EigenGemm.cpp index 0b4220fcbe..674141ed39 100644 --- a/paddle/function/EigenGemm.cpp +++ b/paddle/function/EigenGemm.cpp @@ -77,8 +77,7 @@ struct EigenBlasGemm { } else if (alpha == T(1) && beta == T(1)) { c.device(device) += a.contract(b, dims); } else { - c.device(device) = - c.constant(alpha) * a.contract(b, dims) + c.constant(beta) * c; + c.device(device) = alpha * a.contract(b, dims) + beta * c; } } }; From 29d8825caf921f5349551a18344503345c7b9969 Mon Sep 17 00:00:00 2001 From: dangqingqing Date: Mon, 21 Aug 2017 13:43:51 +0800 Subject: [PATCH 08/13] tune relative precision for unit test img_conv2 in test_NetworkCompare.cpp. 1. It's no problem with relative precision 1e-3 when testing several times in my local machine. 2. But the testing failed with 1e-2 in the TeamCity, and only one value's relative precision is over 1e-2. So tune it to 4e-2 --- paddle/gserver/tests/test_NetworkCompare.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/paddle/gserver/tests/test_NetworkCompare.cpp b/paddle/gserver/tests/test_NetworkCompare.cpp index f930c72fde..d36f72360f 100644 --- a/paddle/gserver/tests/test_NetworkCompare.cpp +++ b/paddle/gserver/tests/test_NetworkCompare.cpp @@ -269,7 +269,8 @@ TEST(Compare, img_conv2) { bool useGpu = FLAGS_use_gpu; double eps = FLAGS_checkgrad_eps; FLAGS_use_gpu = true; - FLAGS_checkgrad_eps = 1e-2; + // Sometimes, this unit test will fail with 1e-2 + FLAGS_checkgrad_eps = 4e-2; compareNetwork(config_file_a, config_file_b); FLAGS_use_gpu = useGpu; FLAGS_checkgrad_eps = eps; From 83abbce8eb750f7e7c844b0959851e901806aa91 Mon Sep 17 00:00:00 2001 From: guosheng Date: Mon, 21 Aug 2017 14:05:56 +0800 Subject: [PATCH 09/13] Follow comments and refine ScaleShiftLayer --- paddle/gserver/layers/ScaleShiftLayer.cpp | 5 +++-- paddle/gserver/tests/test_LayerGrad.cpp | 4 ++-- python/paddle/trainer_config_helpers/layers.py | 5 +++-- .../protostr/test_scale_shift_layer.protostr | 14 +++++++------- .../tests/configs/test_scale_shift_layer.py | 6 ++---- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/paddle/gserver/layers/ScaleShiftLayer.cpp b/paddle/gserver/layers/ScaleShiftLayer.cpp index 4f5b1c6225..06dcb409f8 100644 --- a/paddle/gserver/layers/ScaleShiftLayer.cpp +++ b/paddle/gserver/layers/ScaleShiftLayer.cpp @@ -17,8 +17,9 @@ limitations under the License. */ namespace paddle { /** - * A layer does scaling and shifting to the input by appling a slope and - * an intercept which are trainable to the input element-wise. + * A layer applies a slope and an intercept to the input element-wise for + * scaling and shifting. Noting that this layer is trainable which differs + * from the SlopeInterceptLayer. * * \f[ * y = wx + b diff --git a/paddle/gserver/tests/test_LayerGrad.cpp b/paddle/gserver/tests/test_LayerGrad.cpp index 65429ebada..dd2c955e6a 100644 --- a/paddle/gserver/tests/test_LayerGrad.cpp +++ b/paddle/gserver/tests/test_LayerGrad.cpp @@ -2008,8 +2008,8 @@ TEST(Layer, RowL2NormLayer) { } TEST(Layer, ScaleShiftLayer) { - const size_t batchSize = 128; - const size_t size = 512; + const size_t batchSize = 16; + const size_t size = 32; TestConfig config; config.layerConfig.set_type("scale_shift"); config.layerConfig.set_size(size); diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index 4c7217024a..ec3a87aa36 100755 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -6219,8 +6219,9 @@ def kmax_sequence_score_layer(input, name=None, beam_size=1): @wrap_bias_attr_default() def scale_shift_layer(input, name=None, param_attr=None, bias_attr=None): """ - A layer does scaling and shifting to the input by appling a slope and - an intercept which are trainable to the input element-wise. + A layer applies a slope and an intercept to the input element-wise for + scaling and shifting. Noting that this layer is trainable which differs + from the slope_intercept_layer. .. math:: y = w * x + b diff --git a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_scale_shift_layer.protostr b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_scale_shift_layer.protostr index efaf20f8a7..35ade126a2 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/protostr/test_scale_shift_layer.protostr +++ b/python/paddle/trainer_config_helpers/tests/configs/protostr/test_scale_shift_layer.protostr @@ -14,7 +14,6 @@ layers { input_layer_name: "data" input_parameter_name: "___scale_shift_0__.w0" } - bias_parameter_name: "___scale_shift_0__.wbias" } layers { name: "__scale_shift_1__" @@ -25,6 +24,7 @@ layers { input_layer_name: "data" input_parameter_name: "___scale_shift_1__.w0" } + bias_parameter_name: "___scale_shift_1__.wbias" } parameters { name: "___scale_shift_0__.w0" @@ -37,24 +37,24 @@ parameters { initial_smart: true } parameters { - name: "___scale_shift_0__.wbias" + name: "___scale_shift_1__.w0" size: 1 initial_mean: 0.0 - initial_std: 0.0 + initial_std: 1.0 dims: 1 dims: 1 initial_strategy: 0 - initial_smart: false + initial_smart: true } parameters { - name: "___scale_shift_1__.w0" + name: "___scale_shift_1__.wbias" size: 1 initial_mean: 0.0 - initial_std: 1.0 + initial_std: 0.0 dims: 1 dims: 1 initial_strategy: 0 - initial_smart: true + initial_smart: false } input_layer_names: "data" output_layer_names: "__scale_shift_0__" diff --git a/python/paddle/trainer_config_helpers/tests/configs/test_scale_shift_layer.py b/python/paddle/trainer_config_helpers/tests/configs/test_scale_shift_layer.py index 818d71f15d..dd589116fa 100644 --- a/python/paddle/trainer_config_helpers/tests/configs/test_scale_shift_layer.py +++ b/python/paddle/trainer_config_helpers/tests/configs/test_scale_shift_layer.py @@ -1,11 +1,9 @@ from paddle.trainer_config_helpers import * -settings(batch_size=1000, learning_rate=1e-5) - data = data_layer(name='data', size=100) -scale = scale_shift_layer(input=data) +scale = scale_shift_layer(input=data, bias_attr=False) -scale_shift = scale_shift_layer(input=data, bias_attr=False) +scale_shift = scale_shift_layer(input=data) outputs(scale, scale_shift) From 0af1c4a9feed5a38f34e1ea5a44e3887f702059f Mon Sep 17 00:00:00 2001 From: guosheng Date: Mon, 21 Aug 2017 14:39:05 +0800 Subject: [PATCH 10/13] Follow comments and refine annotations on ScaleShiftLayer --- paddle/gserver/layers/ScaleShiftLayer.cpp | 8 ++++---- python/paddle/trainer_config_helpers/layers.py | 10 +++++++--- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/paddle/gserver/layers/ScaleShiftLayer.cpp b/paddle/gserver/layers/ScaleShiftLayer.cpp index 06dcb409f8..35fd038ab4 100644 --- a/paddle/gserver/layers/ScaleShiftLayer.cpp +++ b/paddle/gserver/layers/ScaleShiftLayer.cpp @@ -17,15 +17,15 @@ limitations under the License. */ namespace paddle { /** - * A layer applies a slope and an intercept to the input element-wise for - * scaling and shifting. Noting that this layer is trainable which differs - * from the SlopeInterceptLayer. + * A layer applies a linear transformation to each element in each row of + * the input matrix. For each element, the layer first re-scale it and then + * adds a bias to it. * * \f[ * y = wx + b * \f] * - * Here, w is scale and b is offset, which are scalars and trainable. + * Here, w is the scale and b is the bias. Both w and b are trainable scalars. * */ diff --git a/python/paddle/trainer_config_helpers/layers.py b/python/paddle/trainer_config_helpers/layers.py index ec3a87aa36..c9e3ded65c 100755 --- a/python/paddle/trainer_config_helpers/layers.py +++ b/python/paddle/trainer_config_helpers/layers.py @@ -6219,9 +6219,13 @@ def kmax_sequence_score_layer(input, name=None, beam_size=1): @wrap_bias_attr_default() def scale_shift_layer(input, name=None, param_attr=None, bias_attr=None): """ - A layer applies a slope and an intercept to the input element-wise for - scaling and shifting. Noting that this layer is trainable which differs - from the slope_intercept_layer. + A layer applies a linear transformation to each element in each row of + the input matrix. For each element, the layer first re-scale it and then + adds a bias to it. + + This layer is very like the SlopeInterceptLayer, except the scale and + bias are trainable. + .. math:: y = w * x + b From 117ce4cbc1a16da1ba8489aaab754aa0ebe5d3ab Mon Sep 17 00:00:00 2001 From: guosheng Date: Mon, 21 Aug 2017 19:23:42 +0800 Subject: [PATCH 11/13] Change class to struct in GemmFunctor to avoid errors on special compilers --- paddle/function/GemmFunctor.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/paddle/function/GemmFunctor.cpp b/paddle/function/GemmFunctor.cpp index dc83278d8e..9e25ee58a1 100644 --- a/paddle/function/GemmFunctor.cpp +++ b/paddle/function/GemmFunctor.cpp @@ -84,7 +84,7 @@ struct BlasGemm { } }; -template class BlasGemm; -template class BlasGemm; +template struct BlasGemm; +template struct BlasGemm; } // namespace paddle From 950dbde56c989f79bace3d53ae38bfae26e84c53 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Mon, 21 Aug 2017 08:41:35 -0700 Subject: [PATCH 12/13] fix rowwise add grad op --- paddle/operators/rowwise_add_op.h | 2 +- python/paddle/v2/framework/tests/test_rowwise_add_op.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/paddle/operators/rowwise_add_op.h b/paddle/operators/rowwise_add_op.h index 232135c38d..771c5d7c0a 100644 --- a/paddle/operators/rowwise_add_op.h +++ b/paddle/operators/rowwise_add_op.h @@ -63,7 +63,7 @@ class RowwiseAddGradKernel : public framework::OpKernel { // https://eigen.tuxfamily.org/dox/unsupported/TensorBase_8h_source.html // colwise add - Eigen::array dims{{1}}; /* dimension to reduce */ + Eigen::array dims{{0}}; /* dimension to reduce */ EigenVector::Flatten(*db).device(place) = OutGrad.sum(dims); } }; diff --git a/python/paddle/v2/framework/tests/test_rowwise_add_op.py b/python/paddle/v2/framework/tests/test_rowwise_add_op.py index 29d72e8500..45d569da29 100644 --- a/python/paddle/v2/framework/tests/test_rowwise_add_op.py +++ b/python/paddle/v2/framework/tests/test_rowwise_add_op.py @@ -20,7 +20,7 @@ class RowwiseAddGradOpTest(GradientChecker): def test_rowwise_add(self): op = create_op("rowwise_add") inputs = { - "X": np.random.uniform(0.1, 1, [10, 10]).astype("float32"), + "X": np.random.uniform(0.1, 1, [5, 10]).astype("float32"), "b": np.random.uniform(0.1, 1, [10]).astype("float32") } self.check_grad(op, inputs, set(["X", "b"]), "Out") From a75a638fb16ac5b08509c3f185d25ec670d3cb12 Mon Sep 17 00:00:00 2001 From: qiaolongfei Date: Mon, 21 Aug 2017 09:13:19 -0700 Subject: [PATCH 13/13] format Copyright --- paddle/operators/rowwise_add_op.h | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/paddle/operators/rowwise_add_op.h b/paddle/operators/rowwise_add_op.h index 771c5d7c0a..1cbd8bb31a 100644 --- a/paddle/operators/rowwise_add_op.h +++ b/paddle/operators/rowwise_add_op.h @@ -1,16 +1,16 @@ /* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. */ +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ #pragma once #include "paddle/framework/eigen.h"