Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into py_reader_doc

distributed_test
sneaxiy 7 years ago
commit 380ab62e4f

File diff suppressed because it is too large Load Diff

@ -293,11 +293,18 @@ class AdamOpKernel : public framework::OpKernel<T> {
auto& grad_tensor = grad_merge.value(); auto& grad_tensor = grad_merge.value();
const T* grad_data = grad_tensor.template data<T>(); const T* grad_data = grad_tensor.template data<T>();
int64_t* rows = nullptr; int64_t* rows = nullptr;
// When compiled without CUDA, the CUDAMutableData() interface should not be
// provided.
#if defined(PADDLE_WITH_CUDA)
if (platform::is_gpu_place(ctx.GetPlace())) { if (platform::is_gpu_place(ctx.GetPlace())) {
rows = grad_merge.mutable_rows()->CUDAMutableData(ctx.GetPlace()); rows = grad_merge.mutable_rows()->CUDAMutableData(ctx.GetPlace());
} else { } else {
#endif
rows = grad_merge.mutable_rows()->data(); rows = grad_merge.mutable_rows()->data();
#if defined(PADDLE_WITH_CUDA)
} }
#endif
auto row_numel = grad_tensor.numel() / grad_merge.rows().size(); auto row_numel = grad_tensor.numel() / grad_merge.rows().size();
SparseAdamFunctor<T> functor( SparseAdamFunctor<T> functor(

@ -106,7 +106,11 @@ class TargetAssignKernel : public framework::OpKernel<T> {
int64_t k = x->dims()[2]; int64_t k = x->dims()[2];
auto x_lod = x->lod().back(); auto x_lod = x->lod().back();
#if defined(PADDLE_WITH_CUDA)
size_t* x_lod_data = x_lod.MutableData(ctx.GetPlace()); size_t* x_lod_data = x_lod.MutableData(ctx.GetPlace());
#else
size_t* x_lod_data = x_lod.data();
#endif
TargetAssignFunctor<T, WT> functor(x_data, match_idx_data, x_lod_data, TargetAssignFunctor<T, WT> functor(x_data, match_idx_data, x_lod_data,
mismatch_value, n, m, p, k, out_data, mismatch_value, n, m, p, k, out_data,
@ -121,7 +125,11 @@ class TargetAssignKernel : public framework::OpKernel<T> {
PADDLE_ENFORCE_EQ(neg_indices->lod().size(), 1UL); PADDLE_ENFORCE_EQ(neg_indices->lod().size(), 1UL);
const int* neg_idx_data = neg_indices->data<int>(); const int* neg_idx_data = neg_indices->data<int>();
auto neg_lod = neg_indices->lod().back(); auto neg_lod = neg_indices->lod().back();
#if defined(PADDLE_WITH_CUDA)
size_t* neg_lod_data = neg_lod.MutableData(ctx.GetPlace()); size_t* neg_lod_data = neg_lod.MutableData(ctx.GetPlace());
#else
size_t* neg_lod_data = neg_lod.data();
#endif
NegTargetAssignFunctor<DeviceContext, T, WT> neg_trg_functor; NegTargetAssignFunctor<DeviceContext, T, WT> neg_trg_functor;
neg_trg_functor(device_ctx, neg_idx_data, neg_lod_data, n, m, k, neg_trg_functor(device_ctx, neg_idx_data, neg_lod_data, n, m, k,
mismatch_value, out_data, out_wt_data); mismatch_value, out_data, out_wt_data);

@ -78,7 +78,7 @@ class LoDTensor2BatchFunctor {
auto lods = lod_tensor.lod(); auto lods = lod_tensor.lod();
PADDLE_ENFORCE_EQ(lods.size(), 1UL, "Only support one level sequence now."); PADDLE_ENFORCE_EQ(lods.size(), 1UL, "Only support one level sequence now.");
auto lod = lods[0]; const auto& lod = lods[0];
std::vector<SeqInfo> seq_info; std::vector<SeqInfo> seq_info;
for (size_t seq_id = 0; seq_id < lod.size() - 1; ++seq_id) { for (size_t seq_id = 0; seq_id < lod.size() - 1; ++seq_id) {

@ -66,7 +66,8 @@ def is_persistable(var):
res = fluid.io.is_persistable(param) res = fluid.io.is_persistable(param)
""" """
if var.desc.type() == core.VarDesc.VarType.FEED_MINIBATCH or \ if var.desc.type() == core.VarDesc.VarType.FEED_MINIBATCH or \
var.desc.type() == core.VarDesc.VarType.FETCH_LIST: var.desc.type() == core.VarDesc.VarType.FETCH_LIST or \
var.desc.type() == core.VarDesc.VarType.READER:
return False return False
return var.persistable return var.persistable

@ -456,11 +456,11 @@ def py_reader(capacity,
name=None, name=None,
use_double_buffer=True): use_double_buffer=True):
""" """
Create a python reader for data feeding in Python Create a Python reader for data feeding in Python
This layer returns a Reader Variable. This layer returns a Reader Variable.
The Reader provides :code:`decorate_paddle_reader` and The Reader provides :code:`decorate_paddle_reader()` and
:code:`decorate_tensor_provider` to set a Python generator as the data :code:`decorate_tensor_provider()` to set a Python generator as the data
source in Python side. When :code:`Executor::Run()` is invoked in C++ source in Python side. When :code:`Executor::Run()` is invoked in C++
side, the data from the generator would be read automatically. Unlike side, the data from the generator would be read automatically. Unlike
:code:`DataFeeder.feed()`, the data reading process and :code:`DataFeeder.feed()`, the data reading process and
@ -561,12 +561,14 @@ def py_reader(capacity,
>>> test_exe = fluid.ParallelExecutor(use_cuda=True, >>> test_exe = fluid.ParallelExecutor(use_cuda=True,
>>> loss_name=test_loss.name, main_program=test_main_prog) >>> loss_name=test_loss.name, main_program=test_main_prog)
>>> for epoch_id in range(10): >>> for epoch_id in range(10):
>>> train_reader.start()
>>> try: >>> try:
>>> while True: >>> while True:
>>> train_exe.run(fetch_list=[train_loss.name]) >>> train_exe.run(fetch_list=[train_loss.name])
>>> except fluid.core.EOFException: >>> except fluid.core.EOFException:
>>> train_reader.reset() >>> train_reader.reset()
>>> >>>
>>> test_reader.start()
>>> try: >>> try:
>>> while True: >>> while True:
>>> test_exe.run(fetch_list=[test_loss.name]) >>> test_exe.run(fetch_list=[test_loss.name])

Loading…
Cancel
Save