qijun
8 years ago
commit
e02cc571cf
@ -0,0 +1,124 @@
|
||||
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License. */
|
||||
|
||||
#pragma once
|
||||
#include "paddle/framework/op_desc.h"
|
||||
#include "paddle/framework/operator.h"
|
||||
|
||||
namespace paddle {
|
||||
namespace framework {
|
||||
|
||||
class GradOpDescMakerBase {
|
||||
public:
|
||||
explicit GradOpDescMakerBase(const OpDescBind& fwd_op) : fwd_op_(fwd_op) {}
|
||||
|
||||
virtual ~GradOpDescMakerBase() = default;
|
||||
virtual std::vector<std::unique_ptr<OpDescBind>> operator()() const = 0;
|
||||
|
||||
protected:
|
||||
static std::vector<std::string> ToGradNames(
|
||||
const std::vector<std::string>& var_names) {
|
||||
std::vector<std::string> ret_val;
|
||||
ret_val.reserve(var_names.size());
|
||||
std::transform(var_names.begin(), var_names.end(),
|
||||
std::back_inserter(ret_val), GradVarName);
|
||||
return ret_val;
|
||||
}
|
||||
|
||||
std::vector<std::string> InputGrad(const std::string& name) const {
|
||||
return ToGradNames(fwd_op_.Input(name));
|
||||
}
|
||||
|
||||
std::vector<std::string> OutputGrad(const std::string& name) const {
|
||||
return ToGradNames(fwd_op_.Output(name));
|
||||
}
|
||||
|
||||
std::vector<std::string> InputNames() const {
|
||||
return this->fwd_op_.InputNames();
|
||||
}
|
||||
|
||||
std::vector<std::string> OutputNames() const {
|
||||
return this->fwd_op_.OutputNames();
|
||||
}
|
||||
|
||||
std::vector<std::string> Input(const std::string& name) const {
|
||||
return fwd_op_.Input(name);
|
||||
}
|
||||
|
||||
std::vector<std::string> Output(const std::string& name) const {
|
||||
return fwd_op_.Output(name);
|
||||
}
|
||||
|
||||
const std::unordered_map<std::string, Attribute>& Attrs() const {
|
||||
return fwd_op_.GetAttrMap();
|
||||
}
|
||||
|
||||
const Attribute& GetAttr(const std::string& name) const {
|
||||
auto& map = fwd_op_.GetAttrMap();
|
||||
auto it = map.find(name);
|
||||
PADDLE_ENFORCE(it != map.end(), "Cannot find attribute %s", name);
|
||||
return it->second;
|
||||
}
|
||||
|
||||
std::string ForwardOpType() const { return this->fwd_op_.Type(); }
|
||||
|
||||
private:
|
||||
const OpDescBind& fwd_op_;
|
||||
};
|
||||
|
||||
class SingleGradOpDescMaker : public GradOpDescMakerBase {
|
||||
public:
|
||||
using GradOpDescMakerBase::GradOpDescMakerBase;
|
||||
|
||||
std::vector<std::unique_ptr<OpDescBind>> operator()() const {
|
||||
std::vector<std::unique_ptr<OpDescBind>> retv;
|
||||
retv.emplace_back(this->Apply());
|
||||
return retv;
|
||||
}
|
||||
|
||||
protected:
|
||||
virtual std::unique_ptr<OpDescBind> Apply() const = 0;
|
||||
};
|
||||
|
||||
class DefaultGradOpDescMaker : public SingleGradOpDescMaker {
|
||||
public:
|
||||
using SingleGradOpDescMaker::SingleGradOpDescMaker;
|
||||
|
||||
protected:
|
||||
virtual std::unique_ptr<OpDescBind> Apply() const {
|
||||
auto* grad = new OpDescBind();
|
||||
grad->SetType(this->GradOpType());
|
||||
|
||||
for (auto& input_param : this->InputNames()) {
|
||||
grad->SetInput(input_param, this->Input(input_param));
|
||||
grad->SetOutput(GradVarName(input_param), this->InputGrad(input_param));
|
||||
}
|
||||
|
||||
for (auto& output_param : this->OutputNames()) {
|
||||
grad->SetInput(output_param, this->Output(output_param));
|
||||
grad->SetInput(GradVarName(output_param), this->OutputGrad(output_param));
|
||||
}
|
||||
|
||||
grad->SetAttrMap(this->Attrs());
|
||||
|
||||
return std::unique_ptr<OpDescBind>(grad);
|
||||
}
|
||||
|
||||
virtual std::string GradOpType() const {
|
||||
return this->ForwardOpType() + "_grad";
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace framework
|
||||
} // namespace paddle
|
||||
@ -1,147 +1,175 @@
|
||||
# Design Doc: LoD (Level-of-Detail) Tensor
|
||||
|
||||
PaddlePaddle's RNN doesn't require that all instances have the same length. To do so, we introduce an extension to Tensor, namely, LoD Tensor.
|
||||
Like other deep learning systems, PaddlePaddle supports training models from sequence data. Also, like other systems, PaddlePaddle represent a mini-batch of sequences as a Tensor. What is different is that PaddlePaddle doesn't require all sequences in a mini-batch to be of the same length. Thus no need for padding zeros.
|
||||
|
||||
## Challenge of Variable-length Inputs
|
||||
| | TensorFlow | PaddlePaddle |
|
||||
|-----------------------|------------|--------------|
|
||||
| RNN | Support | Support |
|
||||
| recursive RNN | Support | Support |
|
||||
| padding zeros | Must | No need |
|
||||
| blob data type | Tensor | LoDTensor |
|
||||
|
||||
People usually represent a mini-batch by a Tensor. For example, a mini-batch of 10 images, each of size 32x32, is a 10x32x32 Tensor. So a transformation, T, of all images can be a matrix multiplication of the 10xOx32-dimensional tensor T and the 10x32x32 Tensor.
|
||||
PaddlePaddle achieves this flexibility by passing through a new data type, *LoD Tensor*, which is a Tensor attached with segmentation index known as *LoD*, between operators. The LoD index doesn't only segment a tensor, but also recursively segments sub-sequences. This document presents the design of LoD and LoDTensor.
|
||||
|
||||
Another example is that each mini-batch contains 32 sentences, where each word is a D-dimensional one-hot vector. If all sentences have the same length L, we can represent this mini-batch by a 32xLxD tensor. However, in most cases, sentences have variable lengths, and we will need an index data structure to record these variable lengths.
|
||||
|
||||
## LoD as a Solution
|
||||
## The Challenge: Variable-length Sequences
|
||||
|
||||
### Mini-Batch of variable-length sentences
|
||||
Most deep learning systems represent a mini-batch as a Tensor. For example, a mini-batch of 10 images, each of size 32x32, is a 10x32x32 Tensor. Another example is that each mini-batch contains N sentences, where each word is a D-dimensional one-hot vector. Suppose that all sentences have the same length L, we can represent this mini-batch by a NxLxD tensor.
|
||||
|
||||
Let's imagine a mini-batch of 3 variable lengths sentences, containing 3, 1, and 2 words respectively. We can represent it by a (3+1+2)xD tensor plus some index information:
|
||||
Both examples show that the elements of sequences are usually of the same size. In the first example, all images are 32x32, and in the second one, all words are D-dimensional vectors. It doesn't make sense to allow variable-sized images, as that would require transformations like convolution to handle variable-sized Tensors.
|
||||
|
||||
The real challenge is that in most cases, sentences have variable lengths, and we will need an index data structure to segment the tensor into sequences. Also, sequences might consist of sub-sequences.
|
||||
|
||||
|
||||
## A Solution: The LoD Index
|
||||
|
||||
To understand our solution, it is best to look at some examples.
|
||||
|
||||
### A Mini-Batch of Sentences
|
||||
|
||||
Let's imagine a mini-batch of 3 variable lengths sentences composed of 3, 1, and 2 words, respectively. We can represent the mini-batch by a (3+1+2)xD tensor plus some index information:
|
||||
|
||||
```
|
||||
3
|
||||
3 1 2
|
||||
||| | ||
|
||||
```
|
||||
|
||||
Each `|` represents a D-dimensional word vectors. The number 3 on top indicate 3 sentences, and numbers 3, 1, and 2 on the second level represent the number of words in each sentence.
|
||||
where each `|` represents a D-dimensional word vector. The numbers, 3, 1, and 2, form a 1-level LoD.
|
||||
|
||||
### Recursive Sequences
|
||||
|
||||
Let check another example of a 2-level LoD Tensor. Consider a mini-batch of three articles with 3, 1, and 2 sentences, and each sentence consists of a variable number of words:
|
||||
|
||||
```
|
||||
3 1 2
|
||||
3 2 4 1 2 3
|
||||
||| || |||| | || |||
|
||||
```
|
||||
|
||||
### Mini-Batch of variable-length videos
|
||||
### A Mini-Batch of Videos
|
||||
|
||||
This approach generalizes to the case where elements are not words, but higher dimensional objects, like images. Suppose that a mini-batch contains videos of the same frame size 640x480. If a mini-batch contains 3 videos of 3, 1, and 2 frames respectively. The underlying tensor is of size (3+1+2)x640x480. The index information illustrates as:
|
||||
LoD tensors generalize to the case where elements are higher dimensional objects, like images. Suppose that a mini-batch contains videos of the same frame size 640x480. Here is a mini-batch of 3 videos with 3, 1, and 2 frames, respectively.
|
||||
|
||||
```
|
||||
3
|
||||
3 1 2
|
||||
口口口 口 口口
|
||||
```
|
||||
|
||||
where each `口` represents an image.
|
||||
The underlying tensor is of size (3+1+2)x640x480, and each `口` represents a 640x480 image.
|
||||
|
||||
### Mini-Batch of fixed-size images
|
||||
### A Mini-Batch of Images
|
||||
|
||||
Let's get back to a typical example, image classification, where each mini-batch has M fixed-sized images. The LoD Tensor representation is
|
||||
In traditional cases like a mini-batch with N fixed-sized images, the LoD Tensor representation is as
|
||||
|
||||
```
|
||||
M
|
||||
1 1 1 1 1
|
||||
口口口口 ... 口
|
||||
```
|
||||
|
||||
The many 1's on the second level seem duplicated. For this particular case of 2 levels and the second level always have length 1, we can ignore the LoD index.
|
||||
|
||||
### Design and summarization
|
||||
In this case, we don't lose any information by ignoring the many 1's in the index and simply considering this LoD Tensor as a usual Tensor:
|
||||
|
||||
In summary, as long as that the essential elements (words or images) have the same size, we can represent mini-batches by a LoD Tensor:
|
||||
```
|
||||
口口口口 ... 口
|
||||
```
|
||||
|
||||
- The underlying tensor has size LxD1xD2x..., where D1xD2... is the size of the essential elements, and
|
||||
- The first dimension size L has an additonal property -- a LoD index as a nested vector:
|
||||
### Model Parameters
|
||||
|
||||
```c++
|
||||
typedef std::vector<std::<vector>> LoD;
|
||||
```
|
||||
A model parameter is just a usual Tensor, which, just like the above example, is a **0-level LoD Tensor**.
|
||||
|
||||
- The LoD index is not necessary when there are only two levels and all elements of the second level have length 1.
|
||||
|
||||
## Slicing of LoD Tensor
|
||||
## The LoD Tensor
|
||||
|
||||
Consider that we have a network with three levels of RNN: the top level one handles articles, the second level one handles sentences, and the basic level one handles words. This network requires that mini-batches represented by 3 level LoD Tensor, for example,
|
||||
Let us revisit above example of the 2-level LoD Tensor
|
||||
|
||||
```
|
||||
3
|
||||
3 1 2
|
||||
3 2 4 1 2 3
|
||||
||| || |||| | || |||
|
||||
```
|
||||
|
||||
To allow each level of RNN to handle its input, we define **the slicing of a LoD Tensor is defined as getting the j-th sequence on level i, or the <i,j>-slice**
|
||||
It is indeed a tree, where leaves are elementary sequences identified by **branches**.
|
||||
|
||||
For example, the third sentence in above example is identified by branch <0,2>, where 0 indicates the first article with length 3, and 2 indicates the third sentence in this article with length 4.
|
||||
|
||||
### The LoD Index
|
||||
|
||||
For example, the <2,1>-slice of above slice is
|
||||
We can save the LoD index in the above example
|
||||
|
||||
```
|
||||
2
|
||||
||
|
||||
3 1 2
|
||||
3 2 4 1 2 3
|
||||
```
|
||||
|
||||
and the <1,2>-slice of above example is
|
||||
in a not-full 2D matrix:
|
||||
|
||||
```c++
|
||||
typedef std::vector<std::vector<int> > LoD;
|
||||
```
|
||||
2
|
||||
2 3
|
||||
|| |||
|
||||
```
|
||||
|
||||
Let's go on slicing this slice. Its <1,1>-slice is
|
||||
where
|
||||
|
||||
- `LoD.size()` is the number of levels, or the maximum length of branches,
|
||||
- `LoD[i][j]` is the length of the j-th segment at the i-th level.
|
||||
|
||||
## The Offset Representation
|
||||
|
||||
To quickly access elementary sequences, we adopt an offset representation -- instead of saving the lengths, we save the beginning and ending elements of sequences.
|
||||
|
||||
In the above example, we accumulate the length of elementary sequences:
|
||||
|
||||
```
|
||||
1
|
||||
1
|
||||
|
|
||||
3 2 4 1 2 3
|
||||
```
|
||||
|
||||
### The Slicing Algorithm
|
||||
into offsets
|
||||
|
||||
The algorithm, with over-simplified data structure, is defined as
|
||||
```
|
||||
0 3 5 9 10 12 15
|
||||
= = = = = =
|
||||
3 2+3 4+5 1+9 2+10 3+12
|
||||
```
|
||||
|
||||
```c++
|
||||
typedef std::vector<std::vector<int>> LoD;
|
||||
so we know that the first sentence is from word 0 to word 3, and the second sentence from work 3 to word 5.
|
||||
|
||||
struct LoDTensor {
|
||||
LoD lod_;
|
||||
float* tensor_;
|
||||
};
|
||||
Similarly, the lengths in the top level LoD
|
||||
|
||||
LoDTensor Slice(const LoDTensor& lodt, int level, int sequence);
|
||||
```
|
||||
3 1 2
|
||||
```
|
||||
|
||||
Let us revisit the example above
|
||||
are transformed into offsets of elements/words as follows:
|
||||
|
||||
```
|
||||
3
|
||||
3 1 2
|
||||
3 2 4 1 2 3
|
||||
||| || |||| | || |||
|
||||
0 9 10 15
|
||||
= = =
|
||||
3+2+4 1+9 2+3+10
|
||||
```
|
||||
|
||||
Suppose that we want to retrieve the <1,2>-slice
|
||||
so we can tell that the first article is from word 0 to word 9, and the second article is from word 9 to word 10.
|
||||
|
||||
The complete offset representation is as follows:
|
||||
|
||||
```
|
||||
2
|
||||
2 3
|
||||
|| |||
|
||||
0 9 10 15
|
||||
0 3 5 9 10 12 15
|
||||
||| || |||| | || |||
|
||||
```
|
||||
|
||||
we will need to find out the starting position of this slice by summing over all leaf nodes in `LoD` to the left of the slice, i.e., 3 + 2 + 4 + 1 = 10.
|
||||
## Slicing of LoD Tensors
|
||||
|
||||
When we use the above 2-level LoD Tensor as the input to a nested-RNN, we need to retrieve certain sequences. Here we define the sequence identified by branch <i,j,...> as the **<i,j,...>-slice**.
|
||||
|
||||
To avoid the traversal of the LoD tree at slicing time, we can do it at the construction time -- instead of saving the lengths of the next level in the LoD tree, we can save the starting offset of the next level. For example, above LoD Tensor can be transformed into
|
||||
For example, the <2>-slice of above example is
|
||||
|
||||
```
|
||||
0
|
||||
0 9 10
|
||||
0 3 5 9 10 12
|
||||
||| || |||| | || |||
|
||||
10 15
|
||||
10 12 15
|
||||
|| |||
|
||||
```
|
||||
|
||||
We don't really need the 0 on top, so the LoD Tensor could be
|
||||
and the <2,0>-slice of above slice is
|
||||
|
||||
```
|
||||
0 9 10
|
||||
0 3 5 9 10 12
|
||||
||| || |||| | || |||
|
||||
10 12
|
||||
||
|
||||
```
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue