Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into develop
zchen0211
8 years ago
commit
58174b12f7
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,118 @@
|
|||||||
|
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License. */
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
#include "paddle/framework/lod_tensor.h"
|
||||||
|
|
||||||
|
namespace paddle {
|
||||||
|
namespace framework {
|
||||||
|
|
||||||
|
/*
|
||||||
|
* DyBatchSeqPosition stores indices of the basic element in tensor. It is used
|
||||||
|
* after lod-tensor's re-assembling, its info can be used to recover the order
|
||||||
|
* in original lod-tensor.
|
||||||
|
*/
|
||||||
|
struct DySeqMeta {
|
||||||
|
size_t begin;
|
||||||
|
size_t end; // not included
|
||||||
|
size_t ori_idx;
|
||||||
|
};
|
||||||
|
|
||||||
|
/*
|
||||||
|
* TensorArray is a C-array-like array of tensors, it is meant to be used with
|
||||||
|
* dynamic iteration primitives such as while_loop. It is used to segment inputs
|
||||||
|
* and store states in all time steps.
|
||||||
|
*
|
||||||
|
* By providing some methods similar to a C++ array, the difinition of some
|
||||||
|
* state-based dynamic models such as RNN cound be more natural and highly
|
||||||
|
* flexible.
|
||||||
|
*/
|
||||||
|
class TensorArray {
|
||||||
|
public:
|
||||||
|
using value_type = float;
|
||||||
|
|
||||||
|
// max number of values allowed to store.
|
||||||
|
const size_t MAX_SIZE{100000};
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Inputs:
|
||||||
|
* - value_shared: share memory between tensors.
|
||||||
|
*/
|
||||||
|
explicit TensorArray(bool values_shared = true)
|
||||||
|
: values_shared_(values_shared) {}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Read the value at location `index` in the `TensorArray`.
|
||||||
|
*/
|
||||||
|
const LoDTensor &Read(size_t index) const;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Write value into the index of the TensorArray.
|
||||||
|
*/
|
||||||
|
void Write(size_t index, const LoDTensor &value);
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Write value into the index of the TensorArray, with memory shared.
|
||||||
|
*/
|
||||||
|
void WriteShared(size_t index, const LoDTensor &value);
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Recover the original LoD-arranged LoDTensor with the `values`, `level` and
|
||||||
|
* `indice_map`.
|
||||||
|
*/
|
||||||
|
LoDTensor Pack(size_t level, const std::vector<DySeqMeta> &meta,
|
||||||
|
const LoD &lod) const;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Split LoDTensor in some `level` and write the generated batches to
|
||||||
|
* `values`, if set `desend`, will sort by length in descending order else in
|
||||||
|
* ascending order.
|
||||||
|
*/
|
||||||
|
std::vector<DySeqMeta> Unpack(const LoDTensor &source, int level,
|
||||||
|
bool length_desend);
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Pack the values into a tensor with rank one higher than each tensor in
|
||||||
|
* values.
|
||||||
|
*/
|
||||||
|
LoDTensor Stack() const;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Unpacks the given division of a rank-`R` tensor into rank-`(R-1)` tensors.
|
||||||
|
*/
|
||||||
|
void Unstack(const LoDTensor &source) const;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Unpacks the given division of a rank-`R` tensor into rank-`(R-1)` tensors,
|
||||||
|
* with memory of tensors shared.
|
||||||
|
*/
|
||||||
|
void UnstackShared(const LoDTensor &source) const;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Return the number of values.
|
||||||
|
*/
|
||||||
|
size_t size() const;
|
||||||
|
|
||||||
|
protected:
|
||||||
|
void Unstack(const LoDTensor &source, bool data_shared) const;
|
||||||
|
|
||||||
|
private:
|
||||||
|
mutable std::vector<LoDTensor> values_;
|
||||||
|
bool values_shared_;
|
||||||
|
}; // class TensorArray
|
||||||
|
|
||||||
|
} // namespace framework
|
||||||
|
} // namespace paddle
|
||||||
@ -0,0 +1,130 @@
|
|||||||
|
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License. */
|
||||||
|
|
||||||
|
#include "paddle/framework/tensor_array.h"
|
||||||
|
|
||||||
|
#include <gtest/gtest.h>
|
||||||
|
|
||||||
|
namespace paddle {
|
||||||
|
namespace framework {
|
||||||
|
|
||||||
|
class TensorArrayTester : public ::testing::Test {
|
||||||
|
protected:
|
||||||
|
void SetUp() override {
|
||||||
|
LoDTensor source;
|
||||||
|
source.Resize(make_ddim({batch_size, dim}));
|
||||||
|
int* data = source.mutable_data<int>(platform::CPUPlace());
|
||||||
|
for (int i = 0; i < 16 * 32; i++) {
|
||||||
|
data[i] = i;
|
||||||
|
}
|
||||||
|
ta.Unstack(source);
|
||||||
|
}
|
||||||
|
|
||||||
|
TensorArray ta;
|
||||||
|
const int batch_size = 16;
|
||||||
|
const int dim = 32;
|
||||||
|
};
|
||||||
|
|
||||||
|
TEST_F(TensorArrayTester, Read) {
|
||||||
|
for (int i = 0; i < batch_size; i++) {
|
||||||
|
const auto& tensor = ta.Read(i);
|
||||||
|
ASSERT_EQ(tensor.dims()[0], 1);
|
||||||
|
ASSERT_EQ(tensor.dims()[1], dim);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST_F(TensorArrayTester, Write) {
|
||||||
|
LoDTensor source;
|
||||||
|
source.Resize(make_ddim({1, dim}));
|
||||||
|
for (int i = 0; i < dim; i++) {
|
||||||
|
*(source.mutable_data<int>(platform::CPUPlace()) + i) = i;
|
||||||
|
}
|
||||||
|
|
||||||
|
ta.Write(2, source);
|
||||||
|
|
||||||
|
const auto& tensor = ta.Read(2);
|
||||||
|
for (int i = 0; i < dim; i++) {
|
||||||
|
EXPECT_EQ(*(tensor.data<int>() + i), *(source.data<int>() + i));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST_F(TensorArrayTester, WriteShared) {
|
||||||
|
LoDTensor source;
|
||||||
|
source.Resize(make_ddim({1, dim}));
|
||||||
|
for (int i = 0; i < dim; i++) {
|
||||||
|
*(source.mutable_data<int>(platform::CPUPlace()) + i) = i;
|
||||||
|
}
|
||||||
|
|
||||||
|
ta.WriteShared(2, source);
|
||||||
|
|
||||||
|
const auto& tensor = ta.Read(2);
|
||||||
|
for (int i = 0; i < dim; i++) {
|
||||||
|
EXPECT_EQ(*(tensor.data<int>() + i), *(source.data<int>() + i));
|
||||||
|
}
|
||||||
|
|
||||||
|
EXPECT_EQ(source.data<int>(), tensor.data<int>());
|
||||||
|
}
|
||||||
|
|
||||||
|
class TensorArrayPackTester : public ::testing::Test {
|
||||||
|
protected:
|
||||||
|
virtual void SetUp() override {
|
||||||
|
lod.push_back(std::vector<size_t>{0, 2, 9, 13});
|
||||||
|
|
||||||
|
source.set_lod(lod);
|
||||||
|
source.Resize(make_ddim({13, 128}));
|
||||||
|
source.mutable_data<int>(platform::CPUPlace());
|
||||||
|
|
||||||
|
// content of each setence: 0 1 2 3 4
|
||||||
|
const auto& level = lod.front();
|
||||||
|
for (size_t i = 0; i < level.size() - 1; i++) {
|
||||||
|
size_t begin = level[i];
|
||||||
|
size_t end = level[i + 1];
|
||||||
|
for (size_t j = begin; j < end; j++) {
|
||||||
|
auto record = source.Slice<int>(j, j + 1);
|
||||||
|
for (int dim = 0; dim < 128; dim++) {
|
||||||
|
record.mutable_data<int>(platform::CPUPlace())[dim] = j - begin;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// unpack
|
||||||
|
meta = ta.Unpack(source, 0, true);
|
||||||
|
}
|
||||||
|
|
||||||
|
LoD lod;
|
||||||
|
TensorArray ta;
|
||||||
|
LoDTensor source;
|
||||||
|
std::vector<DySeqMeta> meta;
|
||||||
|
};
|
||||||
|
|
||||||
|
TEST_F(TensorArrayPackTester, Unpack) {
|
||||||
|
ASSERT_EQ(ta.size(), 7UL);
|
||||||
|
|
||||||
|
const auto& t0 = ta.Read(0);
|
||||||
|
const auto& t1 = ta.Read(1);
|
||||||
|
|
||||||
|
ASSERT_EQ(t0.data<int>()[0], int(0));
|
||||||
|
ASSERT_EQ(t1.data<int>()[0], int(1));
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST_F(TensorArrayPackTester, Pack) {
|
||||||
|
LoDTensor packed = ta.Pack(0, meta, lod);
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST_F(TensorArrayTester, size) {
|
||||||
|
ASSERT_EQ(ta.size(), static_cast<size_t>(batch_size));
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace framework
|
||||||
|
} // namespace paddle
|
||||||
@ -0,0 +1,38 @@
|
|||||||
|
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License. */
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
#include <functional>
|
||||||
|
#include <map>
|
||||||
|
#include "paddle/platform/variant.h"
|
||||||
|
|
||||||
|
namespace paddle {
|
||||||
|
namespace framework {
|
||||||
|
class OperatorBase;
|
||||||
|
using VariableNameMap = std::map<std::string, std::vector<std::string>>;
|
||||||
|
|
||||||
|
// The order should be as same as framework.proto
|
||||||
|
using Attribute =
|
||||||
|
boost::variant<boost::blank, int, float, std::string, std::vector<int>,
|
||||||
|
std::vector<float>, std::vector<std::string>, bool,
|
||||||
|
std::vector<bool>, BlockDesc*>;
|
||||||
|
|
||||||
|
using AttributeMap = std::unordered_map<std::string, Attribute>;
|
||||||
|
|
||||||
|
using OpCreator = std::function<OperatorBase*(
|
||||||
|
const std::string& /*type*/, const VariableNameMap& /*inputs*/,
|
||||||
|
const VariableNameMap& /*outputs*/, const AttributeMap& /*attrs*/)>;
|
||||||
|
|
||||||
|
} // namespace framework
|
||||||
|
} // namespace paddle
|
||||||
Loading…
Reference in new issue