From e5bad90b2876b82500b41df64678de87b470a14f Mon Sep 17 00:00:00 2001 From: superjom Date: Fri, 22 Sep 2017 19:09:42 -0400 Subject: [PATCH 1/3] some enforce change --- paddle/framework/lod_tensor.cc | 26 ++++++++++++++++++-------- paddle/framework/lod_tensor.h | 22 +++++++++++++++------- 2 files changed, 33 insertions(+), 15 deletions(-) diff --git a/paddle/framework/lod_tensor.cc b/paddle/framework/lod_tensor.cc index 908a1f2fd0..88b7f9404a 100644 --- a/paddle/framework/lod_tensor.cc +++ b/paddle/framework/lod_tensor.cc @@ -72,20 +72,30 @@ bool operator==(const LoD& a, const LoD& b) { return true; } +size_t LoDTensor::NumElements(size_t level, size_t idx) const { + PADDLE_ENFORCE_LT(level, NumLevels()); + PADDLE_ENFORCE_LT(idx, NumElements(level)); + // the last level of LoD, just return number of records in Tensor + if (level == NumLevels() - 1) { + } + // high level of LoD, and there is another lower level, return number of + // lower-level elements +} + void LoDTensor::SliceLevels(size_t level_begin, size_t level_end) { auto new_lod = framework::SliceLevels(lod_, level_begin, level_end); lod_ = new_lod; } void LoDTensor::SliceInLevel(size_t level, size_t elem_begin, size_t elem_end) { - PADDLE_ENFORCE(level < NumLevels(), "level [%d] out of range [%d]", level, - NumLevels()); - PADDLE_ENFORCE(elem_begin < NumElements(level), - "element begin [%d] out of range [%d]", elem_begin, - NumElements(level)); - PADDLE_ENFORCE(elem_end < NumElements(level) + 1, - "element end [%d] out of range [%d]", elem_end, - NumElements(level)); + PADDLE_ENFORCE_LT(level, NumLevels(), "level [%d] out of range [%d]", level, + NumLevels()); + PADDLE_ENFORCE_LT(elem_begin, NumElements(level), + "element begin [%d] out of range [%d]", elem_begin, + NumElements(level)); + PADDLE_ENFORCE_LT(elem_end, NumElements(level) + 1, + "element end [%d] out of range [%d]", elem_end, + NumElements(level)); auto new_lod = framework::SliceInLevel(lod_, level, elem_begin, elem_end); lod_ = new_lod; diff --git a/paddle/framework/lod_tensor.h b/paddle/framework/lod_tensor.h index fac5cd20aa..52f29fb5a3 100644 --- a/paddle/framework/lod_tensor.h +++ b/paddle/framework/lod_tensor.h @@ -65,11 +65,8 @@ class LoDTensor : public Tensor { * Get a element from LoD. */ size_t lod_element(size_t level, size_t elem) const { - PADDLE_ENFORCE(level < NumLevels(), "level [%d] out of range [%d]", level, - NumLevels()); - PADDLE_ENFORCE(elem < NumElements(level), - "element begin [%d] out of range [%d]", elem, - NumElements(level)); + PADDLE_ENFORCE_LT(level, NumLevels()); + PADDLE_ENFORCE_LT(elem, NumElements(level)); return (lod_)[level][elem]; } @@ -82,12 +79,23 @@ class LoDTensor : public Tensor { * Number of elements in a level. */ size_t NumElements(size_t level = 0) const { - PADDLE_ENFORCE(level < NumLevels(), "level [%d] out of range [%d]", level, - NumLevels()); + PADDLE_ENFORCE_LT(level, NumLevels()); // the last offset is the end of last element return (lod_)[level].size() - 1; } + /* + * Number of lower-level elements. + * For example, a 2-level lod-tensor + * + * 0-th level | | + * 1-th level || ||| + * + * NumElements(0, 0) get 2 + * NumElements(0, 1) get 3 + */ + size_t NumElements(size_t level, size_t idx) const; + /* * Slice of levels[level_begin:level_end] */ From dcf4682be061ecac26a2cdbb4ff96f716772e154 Mon Sep 17 00:00:00 2001 From: superjom Date: Fri, 22 Sep 2017 20:15:49 -0400 Subject: [PATCH 2/3] add LoDTensor::NumElements(id,id) --- paddle/framework/lod_tensor.cc | 6 ++++++ paddle/framework/lod_tensor.h | 12 ++++++++++++ paddle/framework/lod_tensor_test.cc | 6 ++++++ 3 files changed, 24 insertions(+) diff --git a/paddle/framework/lod_tensor.cc b/paddle/framework/lod_tensor.cc index 88b7f9404a..513e63657b 100644 --- a/paddle/framework/lod_tensor.cc +++ b/paddle/framework/lod_tensor.cc @@ -77,9 +77,15 @@ size_t LoDTensor::NumElements(size_t level, size_t idx) const { PADDLE_ENFORCE_LT(idx, NumElements(level)); // the last level of LoD, just return number of records in Tensor if (level == NumLevels() - 1) { + return lod_[level][idx + 1] - lod_[level][idx]; } // high level of LoD, and there is another lower level, return number of // lower-level elements + auto tmp = SliceInLevel(lod_, level, idx, idx + 1); + PADDLE_ENFORCE_GE(tmp.size(), 2); + // there is a 0 as a placeholder stored in LoD, so the number of elements + // equals lod.size() - 1 + return tmp[1].size() - 1; } void LoDTensor::SliceLevels(size_t level_begin, size_t level_end) { diff --git a/paddle/framework/lod_tensor.h b/paddle/framework/lod_tensor.h index 52f29fb5a3..176e1de4d4 100644 --- a/paddle/framework/lod_tensor.h +++ b/paddle/framework/lod_tensor.h @@ -38,6 +38,18 @@ using Vector = thrust::host_vector< T, thrust::system::cuda::experimental::pinned_allocator>; #endif +/* + * 3-level LoD stores + * + * 0 10 20 + * 0 5 10 15 20 + * 0 2 5 7 10 12 15 20 + * + * - in a level, each element indicates offset in the underlying Tensor + * - the first element should be 0 and that indicates that this sequence start + * from 0 + * - each sequence's begin and end(no-inclusive) is level[id, id+1] + */ using LoD = std::vector>; LoD SliceLevels(const LoD& in, size_t level_begin, size_t level_end); diff --git a/paddle/framework/lod_tensor_test.cc b/paddle/framework/lod_tensor_test.cc index 7915326b27..86db9533cf 100644 --- a/paddle/framework/lod_tensor_test.cc +++ b/paddle/framework/lod_tensor_test.cc @@ -56,6 +56,12 @@ TEST_F(LoDTensorTester, NumElements) { ASSERT_EQ(lod_tensor_.NumElements(2), 8UL); } +TEST_F(LoDTensorTester, NumElements2) { + ASSERT_EQ(lod_tensor_.NumElements(0, 0), 2UL); + ASSERT_EQ(lod_tensor_.NumElements(0, 1), 2UL); + ASSERT_EQ(lod_tensor_.NumElements(1, 1), 2UL); +} + TEST_F(LoDTensorTester, SliceLevels) { // slice 1 level for (size_t level = 0; level < 3UL; ++level) { From 75b9c88f169431956d251509a579d18c2c1d34b6 Mon Sep 17 00:00:00 2001 From: superjom Date: Mon, 25 Sep 2017 17:56:32 -0400 Subject: [PATCH 3/3] update --- paddle/framework/lod_tensor_test.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/paddle/framework/lod_tensor_test.cc b/paddle/framework/lod_tensor_test.cc index 86db9533cf..2b35c0bcd2 100644 --- a/paddle/framework/lod_tensor_test.cc +++ b/paddle/framework/lod_tensor_test.cc @@ -63,7 +63,7 @@ TEST_F(LoDTensorTester, NumElements2) { } TEST_F(LoDTensorTester, SliceLevels) { - // slice 1 level + // shrink 1 level for (size_t level = 0; level < 3UL; ++level) { LoDTensor new_lod_tensor = lod_tensor_; new_lod_tensor.SliceLevels(level, level + 1); @@ -71,7 +71,7 @@ TEST_F(LoDTensorTester, SliceLevels) { ASSERT_EQ(new_lod_tensor.NumElements(0), lod_tensor_.NumElements(level)); ASSERT_EQ(new_lod_tensor.data(), lod_tensor_.data()); } - // slice 2 level + // shrink 2 level for (size_t level = 0; level < 2UL; ++level) { LoDTensor new_lod_tensor = lod_tensor_; new_lod_tensor.SliceLevels(level, level + 2);