diff --git a/paddle/fluid/operators/math/CMakeLists.txt b/paddle/fluid/operators/math/CMakeLists.txt
index e5b438cece..55e2ea7601 100644
--- a/paddle/fluid/operators/math/CMakeLists.txt
+++ b/paddle/fluid/operators/math/CMakeLists.txt
@@ -68,6 +68,7 @@ cc_test(selected_rows_functor_test SRCS selected_rows_functor_test.cc DEPS selec
cc_test(im2col_test SRCS im2col_test.cc DEPS im2col)
cc_test(vol2col_test SRCS vol2col_test.cc DEPS vol2col)
cc_test(sequence_padding_test SRCS sequence_padding_test.cc DEPS sequence_padding)
+cc_test(sequence_pooling_test SRCS sequence_pooling_test.cc DEPS sequence_pooling)
if(WITH_GPU)
nv_test(math_function_gpu_test SRCS math_function_test.cu DEPS math_function)
nv_test(selected_rows_functor_gpu_test SRCS selected_rows_functor_test.cu DEPS selected_rows_functor math_function)
diff --git a/paddle/fluid/operators/math/sequence_pooling.cc b/paddle/fluid/operators/math/sequence_pooling.cc
index 235b5405fb..7be8539a7b 100644
--- a/paddle/fluid/operators/math/sequence_pooling.cc
+++ b/paddle/fluid/operators/math/sequence_pooling.cc
@@ -157,6 +157,31 @@ class FirstSeqPoolFunctor {
}
};
+template <typename T>
+class SumSeqPoolGradFunctor {
+ public:
+ void operator()(const platform::CPUDeviceContext& context,
+ const framework::Tensor& out_grad,
+ framework::LoDTensor* in_grad) {
+ auto lod = in_grad->lod()[0];
+ int64_t out_w = out_grad.numel() / out_grad.dims()[0];
+ int64_t in_w = in_grad->numel() / in_grad->dims()[0];
+ PADDLE_ENFORCE(in_w == out_w);
+ const T* out_g_data = out_grad.data<T>();
+ T* in_g_data = in_grad->mutable_data<T>(context.GetPlace());
+ auto blas = math::GetBlas<platform::CPUDeviceContext, T>(context);
+ for (int i = 0; i < static_cast<int>(lod.size()) - 1; ++i) {
+ int64_t h = static_cast<int64_t>(lod[i + 1] - lod[i]);
+ int64_t in_offset = lod[i] * in_w;
+ const T* out_pos = out_g_data + i * out_w;
+ T* in_pos = in_g_data + in_offset;
+ for (int r = 0; r != h; ++r) {
+ blas.VCOPY(in_w, out_pos, in_pos + r * in_w);
+ }
+ }
+ }
+};
+
template <typename T>
class SequencePoolFunctor<platform::CPUDeviceContext, T> {
public:
@@ -231,9 +256,15 @@ class SequencePoolGradFunctor<platform::CPUDeviceContext, T> {
math::SetConstant<platform::CPUDeviceContext, T> functor;
functor(context, in_grad, 0);
}
+
+ if (pooltype == "SUM") {
+ math::SumSeqPoolGradFunctor<T> sum_pool_grad;
+ sum_pool_grad(context, out_grad, in_grad);
+ return;
+ }
+
auto lod = in_grad->lod()[0];
auto& place = *context.eigen_device();
- auto blas = math::GetBlas<platform::CPUDeviceContext, T>(context);
for (int i = 0; i < static_cast<int>(lod.size()) - 1; ++i) {
auto in_g_t = in_grad->Slice(static_cast<int>(lod[i]),
static_cast<int>(lod[i + 1]));
@@ -247,12 +278,6 @@ class SequencePoolGradFunctor<platform::CPUDeviceContext, T> {
if (pooltype == "AVERAGE") {
in_g_e.device(place) = (out_g_e / static_cast<T>(h)).broadcast(bcast);
- } else if (pooltype == "SUM") {
- const T* out_g_data = out_g_t.data<T>();
- T* in_g_data = in_g_t.mutable_data<T>(context.GetPlace());
- for (int r = 0; r != h; ++r) {
- blas.VCOPY(w, out_g_data, in_g_data + r * w);
- }
} else if (pooltype == "SQRT") {
in_g_e.device(place) =
(out_g_e / std::sqrt(static_cast<T>(h))).broadcast(bcast);
diff --git a/paddle/fluid/operators/math/sequence_pooling_test.cc b/paddle/fluid/operators/math/sequence_pooling_test.cc
new file mode 100644
index 0000000000..2bc008dd34
--- /dev/null
+++ b/paddle/fluid/operators/math/sequence_pooling_test.cc
@@ -0,0 +1,126 @@
+/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/fluid/operators/math/sequence_pooling.h"
+#include <gtest/gtest.h>
+#include <vector>
+
+template <typename DeviceContext, typename Place, typename T>
+void TestSequencePoolingSum(const paddle::framework::LoD& lod) {
+ paddle::framework::LoDTensor cpu_out_grad;
+ paddle::framework::LoDTensor cpu_in_grad;
+ paddle::framework::LoDTensor out_grad;
+ paddle::framework::LoDTensor in_grad;
+ const size_t second_dim = 128u;
+
+ // construct out_grad's tensor in cpu
+ const size_t out_first_dim = lod[0].size() - 1;
+ auto out_dims = paddle::framework::make_ddim(
+ {static_cast<int64_t>(out_first_dim), static_cast<int64_t>(second_dim)});
+
+ cpu_out_grad.mutable_data<T>(out_dims, paddle::platform::CPUPlace());
+ for (int64_t i = 0; i < cpu_out_grad.numel(); ++i) {
+ cpu_out_grad.data<T>()[i] = static_cast<T>(i);
+ }
+
+ // copy to dst out_grad
+ auto* place = new Place();
+ DeviceContext* context = new DeviceContext(*place);
+ if (paddle::platform::is_cpu_place(*place)) {
+ out_grad = cpu_out_grad;
+ } else {
+ TensorCopySync(cpu_out_grad, *place, &out_grad);
+ }
+
+ // construct in_grad
+ in_grad.set_lod(lod);
+ auto in_dims = paddle::framework::make_ddim(
+ {static_cast<int64_t>(lod[0].back()), static_cast<int64_t>(second_dim)});
+ in_grad.mutable_data<T>(in_dims, context->GetPlace());
+
+ // check tensor contruction result
+ PADDLE_ENFORCE_EQ(in_grad.dims().size(), out_grad.dims().size());
+ for (int64_t i = 1; i < out_grad.dims().size(); ++i) {
+ PADDLE_ENFORCE_EQ(in_grad.dims()[i], out_grad.dims()[i]);
+ }
+
+ // call functor
+ paddle::operators::math::SequencePoolGradFunctor<DeviceContext, T>()(
+ *context, "SUM", out_grad, &in_grad);
+
+ if (paddle::platform::is_cpu_place(*place)) {
+ cpu_in_grad = in_grad;
+ } else {
+ TensorCopySync(in_grad, paddle::platform::CPUPlace(), &cpu_in_grad);
+ cpu_in_grad.set_lod(in_grad.lod());
+ }
+
+ EXPECT_EQ(in_grad.numel(), lod[0].back() * second_dim);
+ EXPECT_EQ(in_grad.lod(), lod);
+
+ if (paddle::platform::is_cpu_place(*place)) {
+ for (int64_t i = 0; i < in_grad.lod()[0].size() - 1; ++i) {
+ int64_t begin = in_grad.lod()[0][i];
+ int64_t end = in_grad.lod()[0][i + 1];
+ paddle::framework::Tensor tmp = in_grad.Slice(begin, end);
+ for (int64_t j = 0; j != tmp.numel() / second_dim; ++j) {
+ for (int64_t m = 0; m != second_dim; ++m) {
+ EXPECT_EQ(tmp.data<T>()[m + j * second_dim],
+ out_grad.data<T>()[m + i * second_dim]);
+ }
+ }
+ }
+ } else {
+ for (int64_t i = 0; i < cpu_in_grad.lod()[0].size() - 1; ++i) {
+ int64_t begin = cpu_in_grad.lod()[0][i];
+ int64_t end = cpu_in_grad.lod()[0][i + 1];
+ paddle::framework::Tensor tmp = cpu_in_grad.Slice(begin, end);
+ for (int64_t j = 0; j != tmp.numel() / second_dim; ++j) {
+ for (int64_t m = 0; m != second_dim; ++m) {
+ EXPECT_EQ(tmp.data<T>()[m + j * second_dim],
+ cpu_out_grad.data<T>()[m + i * second_dim]);
+ }
+ }
+ }
+ }
+
+ delete place;
+ delete context;
+}
+
+TEST(SequencePoolingGrad, CPU_SUM) {
+ paddle::framework::LoD lod1;
+ lod1.push_back(std::vector<size_t>{0, 10});
+ TestSequencePoolingSum<paddle::platform::CPUDeviceContext,
+ paddle::platform::CPUPlace, float>(lod1);
+
+ paddle::framework::LoD lod2;
+ lod2.push_back(std::vector<size_t>{0, 2, 7, 10});
+ TestSequencePoolingSum<paddle::platform::CPUDeviceContext,
+ paddle::platform::CPUPlace, float>(lod2);
+}
+
+#ifdef PADDLE_WITH_CUDA
+TEST(SequencePoolingGrad, CUDA_SUM) {
+ paddle::framework::LoD lod1;
+ lod1.push_back(std::vector<size_t>{0, 10});
+ TestSequencePoolingSum<paddle::platform::CUDADeviceContext,
+ paddle::platform::CUDAPlace, float>(lod1);
+
+ paddle::framework::LoD lod2;
+ lod2.push_back(std::vector<size_t>{0, 2, 7, 10});
+ TestSequencePoolingSum<paddle::platform::CUDADeviceContext,
+ paddle::platform::CUDAPlace, float>(lod2);
+}
+#endif
diff --git a/python/paddle/fluid/tests/unittests/CMakeLists.txt b/python/paddle/fluid/tests/unittests/CMakeLists.txt
index 7de0ebce06..68e498c6e8 100644
--- a/python/paddle/fluid/tests/unittests/CMakeLists.txt
+++ b/python/paddle/fluid/tests/unittests/CMakeLists.txt
@@ -78,9 +78,9 @@ if(WITH_DISTRIBUTE)
set_tests_properties(test_dist_word2vec PROPERTIES TIMEOUT 200)
py_test_modules(test_dist_se_resnext MODULES test_dist_se_resnext)
set_tests_properties(test_dist_se_resnext PROPERTIES TIMEOUT 1000)
- # TODO: fix this test
- #py_test_modules(test_dist_transformer MODULES test_dist_transformer)
- #set_tests_properties(test_dist_transformer PROPERTIES TIMEOUT 1000)
+
+ py_test_modules(test_dist_transformer MODULES test_dist_transformer)
+ set_tests_properties(test_dist_transformer PROPERTIES TIMEOUT 1000)
endif(NOT APPLE)
py_test_modules(test_dist_transpiler MODULES test_dist_transpiler)
endif()
diff --git a/python/paddle/fluid/tests/unittests/dist_transformer.py b/python/paddle/fluid/tests/unittests/dist_transformer.py
index a2cc574258..ab44954811 100644
--- a/python/paddle/fluid/tests/unittests/dist_transformer.py
+++ b/python/paddle/fluid/tests/unittests/dist_transformer.py
@@ -35,7 +35,7 @@ import paddle
import paddle.fluid as fluid
import paddle.fluid.layers as layers
from paddle.fluid import core
-from test_dist_base import TestDistRunnerBase, runtime_main
+from test_dist_base import TestDistRunnerBase, runtime_main, RUN_STEP
import paddle.compat as cpt
from paddle.compat import long_type
@@ -562,18 +562,12 @@ def train_loop(exe, train_progm, dev_count, sum_cost, avg_cost, lr_scheduler,
for pass_id in six.moves.xrange(TrainTaskConfig.pass_num):
pass_start_time = time.time()
for batch_id, data in enumerate(train_data()):
- if batch_id >= 5:
+ if batch_id >= RUN_STEP:
break
feed_list = []
total_num_token = 0
- #if TrainTaskConfig.local:
- # lr_rate = lr_scheduler.update_learning_rate()
- #for place_id, data_buffer in enumerate(
- # split_data(
- # data, num_part=dev_count)):
-
if TrainTaskConfig.local:
lr_rate = lr_scheduler.update_learning_rate()
@@ -619,12 +613,11 @@ def train_loop(exe, train_progm, dev_count, sum_cost, avg_cost, lr_scheduler,
init = True
# Validate and save the model for inference.
- if batch_id == 0 or batch_id == 4:
- if TrainTaskConfig.val_file_pattern is not None:
- val_avg_cost, val_ppl = test()
- print("[%f]" % val_avg_cost)
- else:
- assert (False)
+ if TrainTaskConfig.val_file_pattern is not None:
+ val_avg_cost, val_ppl = test()
+ print("[%f]" % val_avg_cost)
+ else:
+ assert (False)
#import transformer_reader as reader
@@ -1701,7 +1694,7 @@ class DistTransformer2x2(TestDistRunnerBase):
def run_trainer(self, args):
TrainTaskConfig.use_gpu = args.use_cuda
- sum_cost, avg_cost, predict, token_num, local_lr_scheduler = get_model(
+ sum_cost, avg_cost, predict, token_num, local_lr_scheduler, test_program = get_model(
args.is_dist, not args.sync_mode)
if args.is_dist:
diff --git a/python/paddle/fluid/tests/unittests/test_dist_transformer.py b/python/paddle/fluid/tests/unittests/test_dist_transformer.py
index 47e8dfaf03..25dcccc28d 100644
--- a/python/paddle/fluid/tests/unittests/test_dist_transformer.py
+++ b/python/paddle/fluid/tests/unittests/test_dist_transformer.py
@@ -61,7 +61,8 @@ class TestDistTransformer2x2Sync(TestDistBase):
def test_dist_train(self):
download_files()
- self.check_with_place("dist_transformer.py", delta=1e-5)
+ self.check_with_place(
+ "dist_transformer.py", delta=1e-5, check_error_log=False)
class TestDistTransformer2x2Async(TestDistBase):
@@ -70,7 +71,8 @@ class TestDistTransformer2x2Async(TestDistBase):
def test_dist_train(self):
download_files()
- self.check_with_place("dist_transformer.py", delta=1.0)
+ self.check_with_place(
+ "dist_transformer.py", delta=1.0, check_error_log=False)
if __name__ == "__main__":