Add job=time in trainer, refine cudnn_conv to reduce gpu memory and speed up training. (#218)
* Add benchmark for PaddlePaddle, tensorflow and caffe * ConvProjection to reduce memory for goolenet * Add unit test for ConvProjection. 1. unit test in test_LayerGrad. 2. compare the ConvPorjection and CudnnConvLayer, also compare the concat_layer+img_conv_layer and concat_layer_conv_projection. * Reduce cudnn_conv memory and add benchmark document. 1. Use TmpMatrix as the workspace in cudnn_conv to reduce gpu memory. It reduce lots of memory. 2. Add benchmark document. 3. fix smallnet_mnist_cifar.py in paddle. * Add job=time and refine cudnn_conv to reduce gpu memroy and speed up * Refine cudnn_conv and shared biases operation in concat_layer and mixed_layer. * follow comments * follow comments * Use unique_ptr to prevent memory leaks in CudnnConvLayer.avx_docs
qingqing01
8 years ago
committed by
GitHub
parent
12945b2c90
commit
45c81a414f
@ -0,0 +1,210 @@
|
||||
/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
|
||||
#include "paddle/utils/Stat.h"
|
||||
#include "ConvProjection.h"
|
||||
|
||||
namespace paddle {
|
||||
|
||||
REGISTER_PROJECTION(conv, ConvProjection);
|
||||
|
||||
ThreadLocalD<std::vector<MemoryHandle*>> ConvProjection::convMem_;
|
||||
|
||||
ConvProjection::ConvProjection(const ProjectionConfig& config,
|
||||
ParameterPtr parameter, bool useGpu)
|
||||
: Projection(config, parameter, useGpu) {
|
||||
|
||||
CHECK(useGpu); // only support GPU
|
||||
getConvParams();
|
||||
initCudnn();
|
||||
|
||||
size_t height = filterH_ * filterW_ * channels_ / groups_;
|
||||
size_t width = numFilters_;
|
||||
weight_.reset(new Weight(height, width, parameter));
|
||||
weightOffset_ = height * width / groups_;
|
||||
}
|
||||
|
||||
void ConvProjection::getConvParams() {
|
||||
const ConvConfig &conf = config_.conv_conf();
|
||||
paddingH_ = conf.padding_y();
|
||||
paddingW_ = conf.padding();
|
||||
|
||||
strideH_ = conf.stride_y();
|
||||
strideW_ = conf.stride();
|
||||
|
||||
filterH_ = conf.filter_size_y();
|
||||
filterW_ = conf.filter_size();
|
||||
|
||||
configImgH_ = conf.img_size();
|
||||
configImgW_ = conf.img_size();
|
||||
|
||||
channels_ = conf.channels();
|
||||
numFilters_ = config_.num_filters();
|
||||
|
||||
groups_ = conf.groups();
|
||||
CHECK_EQ(channels_ % groups_, 0);
|
||||
CHECK_EQ(numFilters_ % groups_, 0);
|
||||
}
|
||||
|
||||
void ConvProjection::initCudnn() {
|
||||
hl_create_filter_descriptor(&filterDesc_, channels_, numFilters_,
|
||||
filterH_, filterW_);
|
||||
hl_create_tensor_descriptor(&inputDesc_);
|
||||
hl_create_tensor_descriptor(&outputDesc_);
|
||||
hl_create_convolution_descriptor(&convDesc_, inputDesc_, filterDesc_,
|
||||
paddingH_, paddingW_, strideH_, strideW_);
|
||||
|
||||
// initialize all to default algorithms
|
||||
fwdAlgo_ = 0;
|
||||
bwdFilterAlgo_ = 0;
|
||||
bwdDataAlgo_ = 0;
|
||||
fwdLimitBytes_ = 0;
|
||||
bwdDataLimitBytes_ = 0;
|
||||
bwdFilterLimitBytes_ = 0;
|
||||
workSpaceInBytes_ = 0;
|
||||
|
||||
batchNum_ = 0;
|
||||
isSelectAlgo_ = false;
|
||||
}
|
||||
|
||||
void ConvProjection::reshapeTensorDesc(int batchSize) {
|
||||
hl_tensor_reshape(inputDesc_, batchSize, channels_, imageH_, imageW_,
|
||||
channels_ * imageH_ * imageW_, imageH_ * imageW_,
|
||||
imageW_, 1);
|
||||
hl_reset_convolution_descriptor(convDesc_, inputDesc_, filterDesc_,
|
||||
paddingH_, paddingW_, strideH_, strideW_);
|
||||
|
||||
// The stride between two consecutive images in ConvProjection may not be 1,
|
||||
// for example, in the case of layer ConcatenateLayer2 with two
|
||||
// ConvProjection, the stride is the output_size of layer ConcatenateLayer2.
|
||||
// So the calculation of nStride is different from CudnnConvLayer.
|
||||
// In fact, only "nStride = out_->value->getStride()" is ok.
|
||||
size_t nStride = numFilters_ * outputH_ * outputW_;
|
||||
if (out_->value->isContiguous()) {
|
||||
CHECK_EQ(nStride, out_->value->getWidth());
|
||||
} else {
|
||||
nStride = out_->value->getStride();
|
||||
}
|
||||
|
||||
hl_tensor_reshape(outputDesc_, batchSize, numFilters_, outputH_, outputW_,
|
||||
nStride, outputH_ * outputW_, outputW_, 1);
|
||||
}
|
||||
|
||||
void ConvProjection::reshape(int batchSize) {
|
||||
size_t width = calOutputSize();
|
||||
CHECK_EQ(width, out_->value->getWidth());
|
||||
|
||||
isSelectAlgo_ = (batchSize == batchNum_);
|
||||
batchNum_ = batchSize;
|
||||
|
||||
if (!isSelectAlgo_) {
|
||||
reshapeTensorDesc(batchSize);
|
||||
hl_conv_workspace(inputDesc_, outputDesc_, filterDesc_,
|
||||
convDesc_, &fwdAlgo_, &fwdLimitBytes_,
|
||||
&bwdDataAlgo_, &bwdDataLimitBytes_,
|
||||
&bwdFilterAlgo_, &bwdFilterLimitBytes_);
|
||||
|
||||
size_t maxWorkSpace = 0;
|
||||
maxWorkSpace = std::max(fwdLimitBytes_, bwdDataLimitBytes_);
|
||||
maxWorkSpace = std::max(maxWorkSpace, bwdFilterLimitBytes_);
|
||||
workSpaceInBytes_ = maxWorkSpace;
|
||||
|
||||
|
||||
VLOG(3) << getName() << " Fwd / BwdData / BwdFilter algo: " << fwdAlgo_
|
||||
<< " / " << bwdDataAlgo_
|
||||
<< " / " << bwdFilterAlgo_;
|
||||
}
|
||||
|
||||
isSelectAlgo_ = true;
|
||||
}
|
||||
|
||||
void ConvProjection::forward() {
|
||||
int batchSize = in_->value->getHeight();
|
||||
reshape(batchSize);
|
||||
|
||||
void* workSpace = NULL;
|
||||
if (workSpaceInBytes_ > 0) {
|
||||
workSpace = getSpaceBytes(workSpaceInBytes_);
|
||||
}
|
||||
|
||||
for (int g = 0; g < groups_; ++g) {
|
||||
REGISTER_TIMER_INFO("CudnnConvFwTimer", getName().c_str());
|
||||
|
||||
real *inputData = in_->value->getData() + g * inputOffset_;
|
||||
real *wgtData = weight_->getW()->getData() + g * weightOffset_;
|
||||
real *outData = out_->value->getData() + g * outputOffset_;
|
||||
hl_convolution_forward(inputDesc_, inputData, outputDesc_,
|
||||
outData, filterDesc_, wgtData,
|
||||
convDesc_, workSpace,
|
||||
fwdLimitBytes_, fwdAlgo_);
|
||||
}
|
||||
}
|
||||
|
||||
void ConvProjection::backward(const UpdateCallback& callback) {
|
||||
REGISTER_TIMER_INFO("CudnnConvBpTimer", getName().c_str());
|
||||
|
||||
void* workSpace = NULL;
|
||||
if (workSpaceInBytes_ > 0) {
|
||||
workSpace = getSpaceBytes(workSpaceInBytes_);
|
||||
}
|
||||
|
||||
for (int g = 0; g < groups_; ++g) {
|
||||
real *outGrad = out_->grad->getData() + g * outputOffset_;
|
||||
if (weight_->getWGrad()) {
|
||||
real *inputData = in_->value->getData() + g * inputOffset_;
|
||||
real *weightGrad = weight_->getWGrad()->getData() + g * weightOffset_;
|
||||
hl_convolution_backward_filter(
|
||||
inputDesc_, inputData, outputDesc_, outGrad, filterDesc_,
|
||||
weightGrad, convDesc_, workSpace, bwdFilterLimitBytes_,
|
||||
bwdFilterAlgo_);
|
||||
}
|
||||
|
||||
MatrixPtr preGrad = in_->grad;
|
||||
if (NULL != preGrad) {
|
||||
real *inputGrad = preGrad->getData() + g * inputOffset_;
|
||||
real *wgtData = weight_->getW()->getData() + g* weightOffset_;
|
||||
hl_convolution_backward_data(
|
||||
inputDesc_, inputGrad, outputDesc_, outGrad, filterDesc_,
|
||||
wgtData, convDesc_, workSpace, bwdDataLimitBytes_,
|
||||
bwdDataAlgo_);
|
||||
}
|
||||
}
|
||||
|
||||
weight_->getParameterPtr()->incUpdate(callback);
|
||||
}
|
||||
|
||||
void* ConvProjection::getSpaceBytes(size_t size) {
|
||||
std::vector<MemoryHandle*>& convMem = *convMem_;
|
||||
if (convMem.empty()) {
|
||||
int numDevices = hl_get_device_count();
|
||||
convMem.resize(numDevices);
|
||||
}
|
||||
|
||||
int devId = hl_get_device();
|
||||
MemoryHandle** localMem = &(convMem[devId]);
|
||||
if (NULL == *localMem || size > (*localMem)->getAllocSize()) {
|
||||
*localMem = new GpuMemoryHandle(size);
|
||||
}
|
||||
return (*localMem)->getBuf();
|
||||
}
|
||||
|
||||
ConvProjection::~ConvProjection() {
|
||||
hl_destroy_tensor_descriptor(inputDesc_);
|
||||
hl_destroy_tensor_descriptor(outputDesc_);
|
||||
hl_destroy_filter_descriptor(filterDesc_);
|
||||
hl_destroy_convolution_descriptor(convDesc_);
|
||||
}
|
||||
|
||||
} // namespace paddle
|
||||
@ -0,0 +1,125 @@
|
||||
/* Copyright (c) 2016 Baidu, Inc. All Rights Reserve.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "Projection.h"
|
||||
|
||||
namespace paddle {
|
||||
|
||||
/**
|
||||
* @brief Convolution projection do the same calculation with CudnnConvLayer.
|
||||
*/
|
||||
class ConvProjection : public Projection {
|
||||
public:
|
||||
/**
|
||||
* Constructor.
|
||||
*/
|
||||
ConvProjection(const ProjectionConfig& config, ParameterPtr parameter,
|
||||
bool useGpu);
|
||||
|
||||
~ConvProjection();
|
||||
|
||||
virtual void forward();
|
||||
virtual void backward(const UpdateCallback& callback);
|
||||
|
||||
protected:
|
||||
void getConvParams();
|
||||
void initCudnn();
|
||||
|
||||
void reshapeTensorDesc(int batchSize);
|
||||
void reshape(int batchSize);
|
||||
|
||||
int outputSize(int imageSize, int filterSize, int padding, int stride) {
|
||||
return (imageSize - filterSize + 2 * padding) / stride + 1;
|
||||
}
|
||||
|
||||
size_t calOutputSize() {
|
||||
imageH_ = in_->getFrameHeight();
|
||||
imageW_ = in_->getFrameWidth();
|
||||
if (imageH_ == 0) imageH_ = configImgH_;
|
||||
if (imageW_ == 0) imageW_ = configImgW_;
|
||||
outputH_ = outputSize(imageH_, filterH_, paddingH_, strideH_);
|
||||
outputW_ = outputSize(imageW_, filterW_, paddingW_, strideW_);
|
||||
|
||||
const_cast<Argument*>(out_)->setFrameHeight(outputH_);
|
||||
const_cast<Argument*>(out_)->setFrameWidth(outputW_);
|
||||
|
||||
inputOffset_ = (channels_ / groups_) * imageH_ * imageW_;
|
||||
outputOffset_ = (numFilters_ / groups_) * outputH_ * outputW_;
|
||||
return outputH_ * outputW_ * numFilters_;
|
||||
}
|
||||
|
||||
static void* getSpaceBytes(size_t size);
|
||||
|
||||
/// imageH_ and imageW_ is calculated from the input layer.
|
||||
int imageH_, imageW_;
|
||||
/// configImgH_ and configImgW_ is obtained from config.
|
||||
int configImgH_, configImgW_;
|
||||
int outputH_, outputW_;
|
||||
int channels_, numFilters_;
|
||||
int paddingH_, paddingW_;
|
||||
int strideH_, strideW_;
|
||||
int filterH_, filterW_;
|
||||
/// One group offset of input data.
|
||||
int inputOffset_;
|
||||
/// One group offset of output data.
|
||||
int outputOffset_;
|
||||
/// One group offset of weight.
|
||||
int weightOffset_;
|
||||
int groups_;
|
||||
|
||||
/// Cudnn tensor descriptor for input.
|
||||
hl_tensor_descriptor inputDesc_;
|
||||
/// Cudnn tensor descriptor for output.
|
||||
hl_tensor_descriptor outputDesc_;
|
||||
/// Cudnn tensor descriptor for filter.
|
||||
hl_filter_descriptor filterDesc_;
|
||||
/// Cudnn tensor descriptor for a convolution operation.
|
||||
hl_convolution_descriptor convDesc_;
|
||||
|
||||
/// Record the algorithm for forward convolution, which is obtained by cudnn
|
||||
/// api to search the best suited algorithm.
|
||||
int fwdAlgo_;
|
||||
/// Record the algorithm for computing convolution gradient with respect to
|
||||
/// filter coefficients.
|
||||
int bwdFilterAlgo_;
|
||||
/// Record the algorithm for computing convolution gradient with respect to
|
||||
/// the output.
|
||||
int bwdDataAlgo_;
|
||||
/// Amount of GPU memory needed as workspace to be able to execute a
|
||||
/// forward convolution with the specified algo.
|
||||
size_t fwdLimitBytes_;
|
||||
/// Amount of GPU memory needed as workspace to be able to execute a
|
||||
/// backwardFilter with the specified algo.
|
||||
size_t bwdDataLimitBytes_;
|
||||
/// Amount of GPU memory needed as workspace to be able to execute a
|
||||
/// backwardData with the specified algo.
|
||||
size_t bwdFilterLimitBytes_;
|
||||
/// Size of total work space.
|
||||
size_t workSpaceInBytes_;
|
||||
|
||||
/// Whether to call cuDNN api to choose conv algorithm.
|
||||
bool isSelectAlgo_;
|
||||
/// batchNum is used to record batch size. If the batch size is changed,
|
||||
/// the selection algorithm will be called.
|
||||
int batchNum_;
|
||||
bool bias_;
|
||||
|
||||
std::unique_ptr<Weight> weight_;
|
||||
static ThreadLocalD<std::vector<MemoryHandle*>> convMem_;
|
||||
};
|
||||
|
||||
} // namespace paddle
|
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,39 @@
|
||||
#edit-mode: -*- python -*-
|
||||
# Copyright (c) 2016 Baidu, Inc. All Rights Reserved
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from paddle.trainer_config_helpers import *
|
||||
|
||||
settings(batch_size=10)
|
||||
data = data_layer(name ="input", size=8*16*16)
|
||||
conv1 = img_conv_layer(input=data, filter_size=1, filter_size_y=1,
|
||||
num_channels=8,
|
||||
num_filters=16, stride=1,
|
||||
bias_attr=False,
|
||||
act=ReluActivation())
|
||||
conv2 = img_conv_layer(input=data, filter_size=1, filter_size_y=1,
|
||||
num_channels=8,
|
||||
num_filters=16, stride=1,
|
||||
bias_attr=False,
|
||||
act=ReluActivation())
|
||||
|
||||
concat = concat_layer(input=[conv1, conv2])
|
||||
|
||||
conv = img_conv_layer(input=data, filter_size=1, filter_size_y=1,
|
||||
num_channels=8,
|
||||
num_filters=16, stride=1,
|
||||
bias_attr=True,
|
||||
act=LinearActivation())
|
||||
|
||||
outputs(concat, conv)
|
||||
@ -0,0 +1,32 @@
|
||||
#edit-mode: -*- python -*-
|
||||
# Copyright (c) 2016 Baidu, Inc. All Rights Reserved
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from paddle.trainer_config_helpers import *
|
||||
|
||||
settings(batch_size=10)
|
||||
data = data_layer(name ="input", size=8*16*16)
|
||||
proj1 = conv_projection(input=data, filter_size=1, filter_size_y=1,
|
||||
num_channels=8, num_filters=16, stride=1)
|
||||
proj2 = conv_projection(input=data, filter_size=1, filter_size_y=1,
|
||||
num_channels=8, num_filters=16, stride=1)
|
||||
concat = concat_layer(input=[proj1, proj2], bias_attr=False, act=ReluActivation())
|
||||
|
||||
proj = conv_projection(input=data, filter_size=1, filter_size_y=1,
|
||||
num_channels=8, num_filters=16, stride=1)
|
||||
|
||||
with mixed_layer(bias_attr=True, act=LinearActivation()) as conv:
|
||||
conv += proj
|
||||
|
||||
outputs(concat, conv)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue