m53297601
/
Paddle
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Compelete max_sequence_len_op (#5913)

Browse Source
release/0.11.0
fengjiayi 8 years ago committed by GitHub
parent 0ac8c74e63
commit 33fa2dfbde
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 121 additions and 8 deletions
Show Stats Download Patch File Download Diff File

2
paddle/operators/CMakeLists.txt
Unescape View File

@ -200,6 +200,7 @@ set(DEPS_OPS
lod_rank_table_op
lod_tensor_to_array_op
array_to_lod_tensor_op
max_sequence_len_op
lstm_op
tensor_array_read_write_op
gru_op
@ -222,6 +223,7 @@ op_library(pool_with_index_op DEPS pooling)
op_library(lod_rank_table_op SRCS lod_rank_table_op.cc DEPS lod_rank_table)
op_library(lod_tensor_to_array_op SRCS lod_tensor_to_array_op.cc DEPS lod_rank_table_op)
op_library(array_to_lod_tensor_op SRCS array_to_lod_tensor_op.cc DEPS lod_rank_table_op)
op_library(max_sequence_len_op SRCS max_sequence_len_op.cc DEPS lod_rank_table)
op_library(tensor_array_read_write_op SRCS tensor_array_read_write_op.cc)
if(WITH_GPU)
op_library(nccl_op DEPS nccl_common)

66
paddle/operators/max_sequence_len_op.cc
Unescape View File

@ -0,0 +1,66 @@
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include "paddle/framework/lod_rank_table.h"
#include "paddle/framework/op_registry.h"
#include "paddle/framework/operator.h"
namespace paddle {
namespace operators {
class MaxSeqenceLenOp : public framework::OperatorBase {
public:
MaxSeqenceLenOp(const std::string &type,
const framework::VariableNameMap &inputs,
const framework::VariableNameMap &outputs,
const framework::AttributeMap &attrs)
: OperatorBase(type, inputs, outputs, attrs) {}
void Run(const framework::Scope &scope,
const platform::DeviceContext &dev_ctx) const override {
auto &rank_table =
scope.FindVar(Input("RankTable"))->Get<framework::LoDRankTable>();
auto *out =
scope.FindVar(Output("Out"))->GetMutable<framework::LoDTensor>();
int64_t *out_ptr = out->mutable_data<int64_t>({1}, platform::CPUPlace());
*out_ptr = rank_table.items()[0].length;
}
};
class MaxSeqenceLenOpProtoMaker : public framework::OpProtoAndCheckerMaker {
public:
MaxSeqenceLenOpProtoMaker(framework::OpProto *proto,
framework::OpAttrChecker *op_checker)
: OpProtoAndCheckerMaker(proto, op_checker) {
AddInput("RankTable", "The lod_rank_table.");
AddOutput("Out", "The max sequence length.");
AddComment(
R"DOC(Calculate the max sequence length through lod_rank_table.)DOC");
}
};
class MaxSeqenceLenInferShape : public framework::InferShapeBase {
public:
void operator()(framework::InferShapeContext *context) const override {
PADDLE_ENFORCE(context->HasInput("RankTable"));
context->SetOutputDim("Out", {1});
}
};
} // namespace operators
} // namespace paddle
REGISTER_OPERATOR(max_sequence_len, paddle::operators::MaxSeqenceLenOp,
paddle::operators::MaxSeqenceLenOpProtoMaker,
paddle::operators::MaxSeqenceLenInferShape,
paddle::framework::EmptyGradOpMaker);

14
python/paddle/v2/fluid/layers.py
Unescape View File

@ -1354,6 +1354,20 @@ def lod_rank_table(x, level=0, main_program=None):
return table
def max_sequence_len(rank_table, main_program=None):
"""
This function creates an operator to calculate the length of
max seqence through input rank_table(should be a lod_rank_table)
"""
helper = LayerHelper("max_seqence_len", **locals())
res = helper.create_tmp_variable(dtype="int64")
helper.append_op(
type="max_sequence_len",
inputs={"RankTable": rank_table},
outputs={"Out": res})
return res
def topk(input, k, main_program=None, startup_program=None):
helper = LayerHelper('topk', **locals())
topk_out = helper.create_tmp_variable(dtype=input.data_type)

47
python/paddle/v2/fluid/tests/test_lod_tensor_array_ops.py
Unescape View File

@ -18,7 +18,11 @@ class TestCPULoDTensorArrayOps(unittest.TestCase):
tensor.set_lod([[0, 3, 9, 10]])
expect = map(lambda x: numpy.array(x).astype('int32'),
[[3, 0, 9], [4, 1], [5, 2], [6], [7], [8]])
self.main(tensor=tensor, expect_array=expect, expect_lod=[] * 6)
self.main(
tensor=tensor,
expect_array=expect,
expect_lod=[] * 6,
expect_max_len=6)
def test_lod_tensor_to_array_level_0_empty_seq(self):
tensor = core.LoDTensor()
@ -27,7 +31,11 @@ class TestCPULoDTensorArrayOps(unittest.TestCase):
tensor.set_lod([[0, 3, 9, 9, 10]])
expect = map(lambda x: numpy.array(x).astype('int32'),
[[3, 0, 9], [4, 1], [5, 2], [6], [7], [8]])
self.main(tensor=tensor, expect_array=expect, expect_lod=[] * 6)
self.main(
tensor=tensor,
expect_array=expect,
expect_lod=[] * 6,
expect_max_len=6)
def test_lod_tensor_to_array_level_1(self):
tensor = core.LoDTensor()
@ -44,7 +52,11 @@ class TestCPULoDTensorArrayOps(unittest.TestCase):
]
lod = [[[0, 2, 5]], [[0, 6, 12]], [[0, 3]]]
self.main(tensor=tensor, expect_array=expect, expect_lod=lod)
self.main(
tensor=tensor,
expect_array=expect,
expect_lod=lod,
expect_max_len=3)
def test_lod_tensor_to_array_level_1_empty_seq(self):
tensor = core.LoDTensor()
@ -63,7 +75,11 @@ class TestCPULoDTensorArrayOps(unittest.TestCase):
]
lod = [[[0, 5, 8, 8, 15]], [[0, 2, 6, 7, 8]], [[0, 2, 6]], [[0, 2]]]
self.main(tensor=tensor, expect_array=expect, expect_lod=lod)
self.main(
tensor=tensor,
expect_array=expect,
expect_lod=lod,
expect_max_len=4)
def test_lod_tensor_to_array_level_2(self):
tensor = core.LoDTensor()
@ -80,7 +96,11 @@ class TestCPULoDTensorArrayOps(unittest.TestCase):
]
lod = [[[0, 1, 3, 4], [0, 1, 4, 8, 12]],
[[0, 4, 7], [0, 1, 5, 9, 17, 21, 27, 31]], [[0, 2], [0, 6, 7]]]
self.main(tensor=tensor, expect_array=expect, expect_lod=lod)
self.main(
tensor=tensor,
expect_array=expect,
expect_lod=lod,
expect_max_len=3)
def test_lod_tensor_to_array_level_2_skip_level(self):
tensor = core.LoDTensor()
@ -88,14 +108,21 @@ class TestCPULoDTensorArrayOps(unittest.TestCase):
numpy.arange(50).reshape(50, 1).astype('int32'), self.place())
tensor.set_lod([[0, 2, 5, 6], [0, 2, 5, 6, 10, 12, 13],
[0, 3, 7, 11, 17, 21, 22, 23, 27, 31, 39, 45, 46, 50]])
self.main(tensor=tensor, expect_array=None, expect_lod=None, level=1)
def main(self, tensor, expect_array, expect_lod, level=0):
self.main(
tensor=tensor,
expect_array=None,
expect_lod=None,
expect_max_len=4,
level=1)
def main(self, tensor, expect_array, expect_lod, expect_max_len, level=0):
place = self.place()
program = Program()
x = layers.data(name='x', shape=[10], main_program=program)
x.persistable = True
table = layers.lod_rank_table(x, level=level, main_program=program)
max_len = layers.max_sequence_len(table, main_program=program)
max_len.persistable = True
array = layers.lod_tensor_to_array(x, table, main_program=program)
array.persistable = True
@ -110,6 +137,10 @@ class TestCPULoDTensorArrayOps(unittest.TestCase):
self.check_array_same(array, expect_array, expect_lod)
self.check_tensor_same(scope.find_var(result.name).get_tensor(), tensor)
self.assertEqual(
numpy.array(scope.find_var(max_len.name).get_tensor())[0],
expect_max_len)
def check_array_same(self, array, expect_tensor, expect_lod):
self.assertEqual(len(expect_tensor), len(array))
for i, exp in enumerate(zip(expect_tensor, expect_lod)):

Write Preview
Loading…
Cancel
Save