Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into add_ut_for_trt

cmake/generic.cmake

@@ -211,7 +211,7 @@ function(merge_static_libs TARGET_NAME)
       set(libfiles ${libfiles} $<TARGET_FILE:${lib}>)
       #endif()
     endforeach()
-   
+
     # windows cmd return error in clean env.
     # COMMAND del "${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_BUILD_TYPE}/${TARGET_NAME}.lib"
     add_custom_command(TARGET ${TARGET_NAME} POST_BUILD
@@ -255,7 +255,7 @@ function(cc_library TARGET_NAME)
       target_link_libraries(${TARGET_NAME} ${cc_library_DEPS})
       add_dependencies(${TARGET_NAME} ${cc_library_DEPS})
     endif()
-    
+
     # cpplint code style
     foreach(source_file ${cc_library_SRCS})
       string(REGEX REPLACE "\\.[^.]*$" "" source ${source_file})
@@ -298,11 +298,10 @@ function(cc_test TARGET_NAME)
              WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
     if (${cc_test_SERIAL})
         set_property(TEST ${TARGET_NAME} PROPERTY RUN_SERIAL 1)
-
+    endif()
     set_property(TEST ${TARGET_NAME} PROPERTY ENVIRONMENT FLAGS_cpu_deterministic=true)
     set_property(TEST ${TARGET_NAME} PROPERTY ENVIRONMENT FLAGS_init_allocated_mem=true)
     set_property(TEST ${TARGET_NAME} PROPERTY ENVIRONMENT FLAGS_cudnn_deterministic=true)
-    endif()
   endif()
 endfunction(cc_test)
 
@@ -366,11 +365,10 @@ function(nv_test TARGET_NAME)
     add_test(${TARGET_NAME} ${TARGET_NAME})
     if (nv_test_SERIAL)
         set_property(TEST ${TARGET_NAME} PROPERTY RUN_SERIAL 1)
-
+    endif()
     set_property(TEST ${TARGET_NAME} PROPERTY ENVIRONMENT FLAGS_cpu_deterministic=true)
     set_property(TEST ${TARGET_NAME} PROPERTY ENVIRONMENT FLAGS_init_allocated_mem=true)
     set_property(TEST ${TARGET_NAME} PROPERTY ENVIRONMENT FLAGS_cudnn_deterministic=true)
-    endif()
   endif()
 endfunction(nv_test)
 
@@ -558,26 +556,26 @@ function(paddle_protobuf_generate_cpp SRCS HDRS)
   set(${HDRS})
 
   if (MOBILE_INFERENCE)
-      set(EXTRA_FLAG "lite:")  
+      set(EXTRA_FLAG "lite:")
   else()
-      set(EXTRA_FLAG "") 
+      set(EXTRA_FLAG "")
   endif()
 
   foreach(FIL ${ARGN})
     get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
     get_filename_component(FIL_WE ${FIL} NAME_WE)
-    
+
     set(_protobuf_protoc_src "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc")
     set(_protobuf_protoc_hdr "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h")
     list(APPEND ${SRCS} "${_protobuf_protoc_src}")
     list(APPEND ${HDRS} "${_protobuf_protoc_hdr}")
-    
+
     add_custom_command(
       OUTPUT "${_protobuf_protoc_src}"
              "${_protobuf_protoc_hdr}"
 
       COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_CURRENT_BINARY_DIR}"
-      COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} 
+      COMMAND ${PROTOBUF_PROTOC_EXECUTABLE}
       -I${CMAKE_CURRENT_SOURCE_DIR}
       --cpp_out "${EXTRA_FLAG}${CMAKE_CURRENT_BINARY_DIR}" ${ABS_FIL}
       DEPENDS ${ABS_FIL} protoc
@@ -646,7 +644,7 @@ function(grpc_library TARGET_NAME)
   get_filename_component(PROTO_PATH ${ABS_PROTO} PATH)
 
   #FIXME(putcn): the follwoing line is supposed to generate *.pb.h and cc, but
-  # somehow it didn't. line 602 to 604 is to patching this. Leaving this here 
+  # somehow it didn't. line 602 to 604 is to patching this. Leaving this here
   # for now to enable dist CI.
   protobuf_generate_cpp(grpc_proto_srcs grpc_proto_hdrs "${ABS_PROTO}")
   set(grpc_grpc_srcs "${CMAKE_CURRENT_BINARY_DIR}/${PROTO_WE}.grpc.pb.cc")

doc/fluid/api/initializer.rst

@@ -32,6 +32,15 @@ Normal
     :members:
     :noindex:
 
+.. _api_fluid_initializer_Normal:
+
+TruncatedNormal
+------
+
+..  autoclass:: paddle.fluid.initializer.TruncatedNormal
+    :members:
+    :noindex:
+
 .. _api_fluid_initializer_Xavier:
 
 Xavier

doc/fluid/api/layers.rst

@@ -290,14 +290,6 @@ Recv
 ..  autofunction:: paddle.fluid.layers.Recv
     :noindex:
 
-.. _api_fluid_layers_open_recordio_file:
-
-open_recordio_file
-------------------
-
-..  autofunction:: paddle.fluid.layers.open_recordio_file
-    :noindex:
-
 .. _api_fluid_layers_open_files:
 
 open_files

paddle/fluid/API.spec

@@ -1,17 +1,10 @@
 paddle.fluid.Program.__init__ ArgSpec(args=['self'], varargs=None, keywords=None, defaults=None)
 paddle.fluid.Program.block ArgSpec(args=['self', 'index'], varargs=None, keywords=None, defaults=None)
 paddle.fluid.Program.clone ArgSpec(args=['self', 'for_test'], varargs=None, keywords=None, defaults=(False,))
-paddle.fluid.Program.copy_data_info_from ArgSpec(args=['self', 'other'], varargs=None, keywords=None, defaults=None)
-paddle.fluid.Program.create_block ArgSpec(args=['self', 'parent_idx'], varargs=None, keywords=None, defaults=(None,))
 paddle.fluid.Program.current_block ArgSpec(args=['self'], varargs=None, keywords=None, defaults=None)
-paddle.fluid.Program.get_desc ArgSpec(args=['self'], varargs=None, keywords=None, defaults=None)
 paddle.fluid.Program.global_block ArgSpec(args=['self'], varargs=None, keywords=None, defaults=None)
-paddle.fluid.Program.inference_optimize ArgSpec(args=['self', 'export_for_deployment'], varargs=None, keywords=None, defaults=(True,))
 paddle.fluid.Program.list_vars ArgSpec(args=['self'], varargs=None, keywords=None, defaults=None)
-paddle.fluid.Program.optimized_guard ArgSpec(args=[], varargs='args', keywords='kwds', defaults=None)
 paddle.fluid.Program.parse_from_string ArgSpec(args=['binary_str'], varargs=None, keywords=None, defaults=None)
-paddle.fluid.Program.prune ArgSpec(args=['self', 'targets'], varargs=None, keywords=None, defaults=None)
-paddle.fluid.Program.rollback ArgSpec(args=['self'], varargs=None, keywords=None, defaults=None)
 paddle.fluid.Program.to_string ArgSpec(args=['self', 'throw_on_error', 'with_details'], varargs=None, keywords=None, defaults=(False,))
 paddle.fluid.Operator.__init__ ArgSpec(args=['self', 'block', 'desc', 'type', 'inputs', 'outputs', 'attrs'], varargs=None, keywords=None, defaults=(None, None, None, None))
 paddle.fluid.Operator.all_attrs ArgSpec(args=['self'], varargs=None, keywords=None, defaults=None)
@@ -86,6 +79,7 @@ paddle.fluid.io.get_inference_program ArgSpec(args=['target_vars', 'main_program
 paddle.fluid.initializer.ConstantInitializer.__init__ ArgSpec(args=['self', 'value', 'force_cpu'], varargs=None, keywords=None, defaults=(0.0, False))
 paddle.fluid.initializer.UniformInitializer.__init__ ArgSpec(args=['self', 'low', 'high', 'seed'], varargs=None, keywords=None, defaults=(-1.0, 1.0, 0))
 paddle.fluid.initializer.NormalInitializer.__init__ ArgSpec(args=['self', 'loc', 'scale', 'seed'], varargs=None, keywords=None, defaults=(0.0, 1.0, 0))
+paddle.fluid.initializer.TruncatedNormalInitializer.__init__ ArgSpec(args=['self', 'loc', 'scale', 'seed'], varargs=None, keywords=None, defaults=(0.0, 1.0, 0))
 paddle.fluid.initializer.XavierInitializer.__init__ ArgSpec(args=['self', 'uniform', 'fan_in', 'fan_out', 'seed'], varargs=None, keywords=None, defaults=(True, None, None, 0))
 paddle.fluid.initializer.BilinearInitializer.__init__ ArgSpec(args=['self'], varargs=None, keywords=None, defaults=None)
 paddle.fluid.initializer.MSRAInitializer.__init__ ArgSpec(args=['self', 'uniform', 'fan_in', 'seed'], varargs=None, keywords=None, defaults=(True, None, 0))
@@ -131,7 +125,7 @@ paddle.fluid.layers.split ArgSpec(args=['input', 'num_or_sections', 'dim', 'name
 paddle.fluid.layers.ctc_greedy_decoder ArgSpec(args=['input', 'blank', 'name'], varargs=None, keywords=None, defaults=(None,))
 paddle.fluid.layers.edit_distance ArgSpec(args=['input', 'label', 'normalized', 'ignored_tokens'], varargs=None, keywords=None, defaults=(True, None))
 paddle.fluid.layers.l2_normalize ArgSpec(args=['x', 'axis', 'epsilon', 'name'], varargs=None, keywords=None, defaults=(1e-12, None))
-paddle.fluid.layers.matmul ArgSpec(args=['x', 'y', 'transpose_x', 'transpose_y', 'name'], varargs=None, keywords=None, defaults=(False, False, None))
+paddle.fluid.layers.matmul ArgSpec(args=['x', 'y', 'transpose_x', 'transpose_y', 'alpha', 'name'], varargs=None, keywords=None, defaults=(False, False, 1.0, None))
 paddle.fluid.layers.topk ArgSpec(args=['input', 'k', 'name'], varargs=None, keywords=None, defaults=(None,))
 paddle.fluid.layers.warpctc ArgSpec(args=['input', 'label', 'blank', 'norm_by_times'], varargs=None, keywords=None, defaults=(0, False))
 paddle.fluid.layers.sequence_reshape ArgSpec(args=['input', 'new_dim'], varargs=None, keywords=None, defaults=None)
@@ -175,9 +169,9 @@ paddle.fluid.layers.stack ArgSpec(args=['x', 'axis'], varargs=None, keywords=Non
 paddle.fluid.layers.pad2d ArgSpec(args=['input', 'paddings', 'mode', 'pad_value', 'data_format', 'name'], varargs=None, keywords=None, defaults=([0, 0, 0, 0], 'constant', 0.0, 'NCHW', None))
 paddle.fluid.layers.unstack ArgSpec(args=['x', 'axis', 'num'], varargs=None, keywords=None, defaults=(0, None))
 paddle.fluid.layers.sequence_enumerate ArgSpec(args=['input', 'win_size', 'pad_value', 'name'], varargs=None, keywords=None, defaults=(0, None))
+paddle.fluid.layers.expand ArgSpec(args=['x', 'expand_times', 'name'], varargs=None, keywords=None, defaults=(None,))
 paddle.fluid.layers.sequence_concat ArgSpec(args=['input', 'name'], varargs=None, keywords=None, defaults=(None,))
 paddle.fluid.layers.data ArgSpec(args=['name', 'shape', 'append_batch_size', 'dtype', 'lod_level', 'type', 'stop_gradient'], varargs=None, keywords=None, defaults=(True, 'float32', 0, VarType.LOD_TENSOR, True))
-paddle.fluid.layers.open_recordio_file ArgSpec(args=['filename', 'shapes', 'lod_levels', 'dtypes', 'pass_num', 'for_parallel'], varargs=None, keywords=None, defaults=(1, True))
 paddle.fluid.layers.open_files ArgSpec(args=['filenames', 'shapes', 'lod_levels', 'dtypes', 'thread_num', 'buffer_size', 'pass_num', 'is_test'], varargs=None, keywords=None, defaults=(None, None, 1, None))
 paddle.fluid.layers.read_file ArgSpec(args=['reader'], varargs=None, keywords=None, defaults=None)
 paddle.fluid.layers.shuffle ArgSpec(args=['reader', 'buffer_size'], varargs=None, keywords=None, defaults=None)

paddle/fluid/framework/data_device_transform.cc

@@ -25,6 +25,10 @@ void TransDataDevice(const Tensor &in, const platform::Place &dst_place,
       in.place().which(), dst_place.which(),
       "Currently, model parallelism is only supported between CPU and CUDA");
 
+  // NOTE(yy): TransDataDevice should wait for computation of input.
+  platform::DeviceContextPool::Instance().Get(in.place())->Wait();
+  platform::DeviceContextPool::Instance().Get(dst_place)->Wait();
+
   // FIXME(zcd): TransDataDevice is used to transform data from GPU to CPU and
   // the enforced checkings have been done in GetDeviceContext, so the
   // `dev_ctx->Wait()` is necessary. But `dev_ctx->Wait()` will make the program

paddle/fluid/framework/grad_op_desc_maker.h

@@ -129,6 +129,9 @@ class GradOpDescMakerBase {
 
   std::string ForwardOpType() const { return this->fwd_op_.Type(); }
 
+ protected:
+  const OpDesc& ForwardOp() const { return fwd_op_; }
+
  private:
   const OpDesc& fwd_op_;
   const std::unordered_set<std::string>& no_grad_set_;

paddle/fluid/framework/prune.cc

@@ -183,28 +183,5 @@ void Prune(const proto::ProgramDesc& input, proto::ProgramDesc* output) {
   output->clear_blocks();
   prune_impl(input, output, 0, -1, &dependent_vars);
 }
-
-void inference_optimize_impl(proto::ProgramDesc* input, int block_id) {
-  auto* op_field = input->mutable_blocks(block_id)->mutable_ops();
-  for (auto& op_desc : *op_field) {
-    for (auto& attr : *op_desc.mutable_attrs()) {
-      if (attr.name() == "is_test") {
-        attr.set_b(true);
-        break;
-      }
-    }
-  }
-}
-
-void InferenceOptimize(const proto::ProgramDesc& input,
-                       proto::ProgramDesc* output) {
-  *output = input;
-  int num_blocks = output->blocks_size();
-  PADDLE_ENFORCE_GT(num_blocks, 0, "ProgramDesc must have at least one block");
-  for (int i = 0; i < num_blocks; ++i) {
-    inference_optimize_impl(output, i);
-  }
-}
-
 }  // namespace framework
 }  // namespace paddle

paddle/fluid/framework/prune.h

@@ -22,8 +22,5 @@ namespace framework {
 
 void Prune(const proto::ProgramDesc& input, proto::ProgramDesc* output);
 
-void InferenceOptimize(const proto::ProgramDesc& input,
-                       proto::ProgramDesc* output);
-
 }  // namespace framework
 }  // namespace paddle

paddle/fluid/operators/array_to_lod_tensor_op.cc

@@ -11,6 +11,7 @@ distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License. */
+#include <paddle/fluid/operators/math/concat.h>
 #include <numeric>
 
 #include "paddle/fluid/framework/lod_rank_table.h"
@@ -24,6 +25,50 @@ namespace operators {
 
 using LoD = framework::LoD;
 
+class ArrayToLoDFunctor;
+template <typename DeviceContext>
+struct ArrayToLoDFunctorImpl {
+  const ArrayToLoDFunctor *prev_functor_;
+  DeviceContext *dev_ctx_;
+
+  template <typename T>
+  void apply();
+};
+
+struct ArrayToLoDFunctor : public boost::static_visitor<void> {
+  std::vector<framework::Tensor> in;
+  mutable framework::Tensor *out;
+
+  template <typename Place>
+  void operator()(Place place) const {
+    auto &pool = platform::DeviceContextPool::Instance();
+    if (std::is_same<Place, platform::CPUPlace>::value) {
+      Apply(static_cast<platform::CPUDeviceContext *>(pool.Get(place)));
+    } else {
+#ifdef PADDLE_WITH_CUDA
+      Apply(static_cast<platform::CUDADeviceContext *>(pool.Get(place)));
+#else
+      PADDLE_THROW("Fluid is not compiled with CUDA");
+#endif
+    }
+  }
+
+  template <typename DeviceContext>
+  void Apply(DeviceContext *dev_ctx) const {
+    ArrayToLoDFunctorImpl<DeviceContext> functor;
+    functor.dev_ctx_ = dev_ctx;
+    functor.prev_functor_ = this;
+    framework::VisitDataType(framework::ToDataType(out->type()), functor);
+  }
+};
+
+template <typename DeviceContext>
+template <typename T>
+void ArrayToLoDFunctorImpl<DeviceContext>::apply() {
+  math::ConcatFunctor<DeviceContext, T> func;
+  func(*dev_ctx_, prev_functor_->in, 0, prev_functor_->out);
+}
+
 class ArrayToLoDTensorOp : public framework::OperatorBase {
  public:
   ArrayToLoDTensorOp(const std::string &type,
@@ -47,14 +92,18 @@ class ArrayToLoDTensorOp : public framework::OperatorBase {
     int rank = x[0].dims().size();
     platform::Place place = x[0].place();
     std::type_index data_type = x[0].type();
-    framework::DDim ins_dims = framework::slice_ddim(x[0].dims(), 1, rank);
     int64_t batch_size = x[0].dims()[0];
+    framework::DDim ins_dims = rank > 1
+                                   ? framework::slice_ddim(x[0].dims(), 1, rank)
+                                   : framework::make_ddim({0});
     for (size_t i = 1; i < x.size(); ++i) {
-      PADDLE_ENFORCE_EQ(framework::slice_ddim(x[i].dims(), 1, rank), ins_dims,
+      auto ins_i_dims = rank > 1 ? framework::slice_ddim(x[i].dims(), 1, rank)
+                                 : framework::make_ddim({0});
+      PADDLE_ENFORCE_EQ(ins_i_dims, ins_dims,
                         "The dimension of the %zu'th element in LoDTensorArray "
                         "differs from previous ones.",
                         i);
-      PADDLE_ENFORCE(platform::places_are_same_class(x[i].place(), place),
+      PADDLE_ENFORCE(x[i].place() == place,
                      "The place class of the %zu'th element in LoDTensorArray "
                      "differs from previous ones.",
                      i);
@@ -82,13 +131,14 @@ class ArrayToLoDTensorOp : public framework::OperatorBase {
     // Build LoDTensor `out`
     framework::LoD *out_lod = out->mutable_lod();
     out_lod->clear();
-    size_t out_offset = 0;
     auto prefix_lod = rank_table.coarse_lod();
     prefix_lod.emplace_back();
     auto &cur_level_lod = prefix_lod.back();
     cur_level_lod.push_back(0);
+    ArrayToLoDFunctor functor;
     for (size_t idx : table_item_idx) {
       cur_level_lod.push_back(cur_level_lod.back() + table_items[idx].length);
+      PADDLE_ENFORCE_LE(table_items[idx].length, x.size());
       for (size_t x_idx = 0; x_idx < table_items[idx].length; ++x_idx) {
         auto lod_and_offset = framework::GetSubLoDAndAbsoluteOffset(
             x[x_idx].lod(), idx, idx + 1, 0);
@@ -106,17 +156,11 @@ class ArrayToLoDTensorOp : public framework::OperatorBase {
         if (len == 0) {
           continue;
         }
-        auto slice = out->Slice(out_offset, out_offset + len);
+        functor.in.emplace_back(x[x_idx].Slice(start_offset, end_offset));
-
-        platform::DeviceContextPool &pool =
-            platform::DeviceContextPool::Instance();
-        auto &dev_ctx = *pool.Get(place);
-
-        framework::TensorCopy(x[x_idx].Slice(start_offset, end_offset), place,
-                              dev_ctx, &slice);
-        out_offset += len;
       }
     }
+    functor.out = out;
+    platform::VisitPlace(place, functor);
     out_lod->insert(out_lod->begin(), prefix_lod.begin(), prefix_lod.end());
   }
 };

paddle/fluid/operators/conv_transpose_op.cc

@@ -29,6 +29,8 @@ void ConvTransposeOp::InferShape(framework::InferShapeContext* ctx) const {
 
   auto in_dims = ctx->GetInputDim("Input");
   auto filter_dims = ctx->GetInputDim("Filter");
+  std::vector<int> output_size =
+      ctx->Attrs().Get<std::vector<int>>("output_size");
   std::vector<int> strides = ctx->Attrs().Get<std::vector<int>>("strides");
   std::vector<int> paddings = ctx->Attrs().Get<std::vector<int>>("paddings");
   std::vector<int> dilations = ctx->Attrs().Get<std::vector<int>>("dilations");
@@ -42,6 +44,10 @@ void ConvTransposeOp::InferShape(framework::InferShapeContext* ctx) const {
   PADDLE_ENFORCE(in_dims.size() - strides.size() == 2U,
                  "ConvTransposeOp input dimension and strides dimension should "
                  "be consistent.");
+  if (output_size.size())
+    PADDLE_ENFORCE_EQ(output_size.size(), strides.size(),
+                      "ConvTransposeOp output_size dimension and strides "
+                      "dimension should be the same.");
   PADDLE_ENFORCE_EQ(paddings.size(), strides.size(),
                     "ConvTransposeOp paddings dimension and strides "
                     "dimension should be the same.");
@@ -55,8 +61,17 @@ void ConvTransposeOp::InferShape(framework::InferShapeContext* ctx) const {
   std::vector<int64_t> output_shape({in_dims[0], filter_dims[1] * groups});
   for (size_t i = 0; i < strides.size(); ++i) {
     auto filter_extent = dilations[i] * (filter_dims[i + 2] - 1) + 1;
-    output_shape.push_back((in_dims[i + 2] - 1) * strides[i] - 2 * paddings[i] +
+    auto infer_shape =
-                           filter_extent);
+        (in_dims[i + 2] - 1) * strides[i] - 2 * paddings[i] + filter_extent;
+    if (output_size.size()) {
+      PADDLE_ENFORCE((output_size[i] >= infer_shape &&
+                      output_size[i] < infer_shape + strides[i]),
+                     "ConvTransposeOp output_size should be "
+                     "in appropriate range.");
+      output_shape.push_back(output_size[i]);
+    } else {
+      output_shape.push_back(infer_shape);
+    }
   }
   ctx->SetOutputDim("Output", framework::make_ddim(output_shape));
 }
@@ -103,6 +118,10 @@ void Conv2DTransposeOpMaker::Make() {
   AddOutput("Output",
             "(Tensor) The output tensor of convolution transpose operator. "
             "The format of output tensor is also NCHW.");
+  AddAttr<std::vector<int>>("output_size",
+                            "(vector<int> default: []), the "
+                            "size of the output tensor")
+      .SetDefault({});
   AddAttr<int>("groups",
                "(int default:1), the groups number of the convolution "
                "transpose operator. ")
@@ -192,7 +211,10 @@ void Conv3DTransposeOpMaker::Make() {
             "Where N is batch size, C is "
             "the number of channels, D is the depth of the feature, H is the "
             "height of the feature, and W is the width of the feature.");
-
+  AddAttr<std::vector<int>>("output_size",
+                            "(vector<int> default: []), the "
+                            "size of the output tensor")
+      .SetDefault({});
   AddAttr<std::vector<int>>(
       "dilations",
       "(vector<int> default:{1, 1, 1}), the "
@@ -247,7 +269,7 @@ Parameters(strides, paddings) are three elements. These three elements represent
 depth, height and width, respectively.
 The input(X) size and output(Out) size may be different.
 
-Example:   
+Example:
   Input:
        Input shape: $(N, C_{in}, D_{in}, H_{in}, W_{in})$
        Filter shape: $(C_{in}, C_{out}, D_f, H_f, W_f)$

paddle/fluid/operators/cross_entropy_op.h

@@ -86,10 +86,10 @@ class XeGradFunctor {
     auto x_is_true_offset = sample_id * num_classes_ + label_[sample_id];
     for (size_t x_offset = sample_id * num_classes_;
          x_offset < (sample_id + 1) * num_classes_; ++x_offset) {
-      dx_[x_offset] =
+      dx_[x_offset] = (x_offset != x_is_true_offset ||
-          (x_offset != x_is_true_offset || label_[sample_id] == ignore_index_)
+                       label_[sample_id] == static_cast<int64_t>(ignore_index_))
-              ? static_cast<T>(0)
+                          ? static_cast<T>(0)
-              : -dy_[sample_id] / x_[x_offset];
+                          : -dy_[sample_id] / x_[x_offset];
     }
   }
 

paddle/fluid/operators/elementwise_mul_op.cc

@@ -13,9 +13,45 @@ See the License for the specific language governing permissions and
 limitations under the License. */
 
 #include "paddle/fluid/operators/elementwise_mul_op.h"
+#include <string>
 #include "paddle/fluid/operators/elementwise_op.h"
+
+namespace paddle {
+namespace operators {
+
+class ElementwiseMulOpGradDescMaker : public framework::SingleGradOpDescMaker {
+ public:
+  using framework::SingleGradOpDescMaker::SingleGradOpDescMaker;
+
+ protected:
+  std::unique_ptr<framework::OpDesc> Apply() const override {
+    std::unique_ptr<framework::OpDesc> op(new framework::OpDesc());
+    op->SetType("elementwise_mul_grad");
+    op->SetInput("X", Input("X"));
+    op->SetInput("Y", Input("Y"));
+    op->SetInput(framework::GradVarName("Out"), OutputGrad("Out"));
+    op->SetAttrMap(Attrs());
+    op->SetOutput(framework::GradVarName("X"), InputGrad("X"));
+    op->SetOutput(framework::GradVarName("Y"), InputGrad("Y"));
+    return op;
+  }
+};
+
+class ElementwiseMulOpMaker : public ElementwiseOpMaker {
+ protected:
+  virtual std::string GetName() const { return "Mul"; }
+  virtual std::string GetEquation() const { return "Out = X \\\\odot Y"; }
+};
+
+}  // namespace operators
+}  // namespace paddle
+
 namespace ops = paddle::operators;
-REGISTER_ELEMWISE_OP(elementwise_mul, "Mul", "Out = X \\\\odot Y");
+REGISTER_OPERATOR(elementwise_mul, ops::ElementwiseOp,
+                  ops::ElementwiseMulOpMaker, ops::ElementwiseOpInferVarType,
+                  ops::ElementwiseMulOpGradDescMaker);
+REGISTER_OPERATOR(elementwise_mul_grad, ops::ElementwiseOpGrad);
+
 REGISTER_OP_CPU_KERNEL(
     elementwise_mul,
     ops::ElementwiseMulKernel<paddle::platform::CPUDeviceContext, float>,

paddle/fluid/operators/elementwise_mul_op.h

@@ -93,8 +93,8 @@ class ElementwiseMulGradKernel : public ElemwiseGradKernel<T> {
 
     auto* x = ctx.Input<Tensor>("X");
     auto* y = ctx.Input<Tensor>("Y");
-    auto* out = ctx.Input<Tensor>("Out");
     auto* dout = ctx.Input<Tensor>(framework::GradVarName("Out"));
+    auto* out = dout;  // out is not necessary
     auto* dx = ctx.Output<Tensor>(framework::GradVarName("X"));
     auto* dy = ctx.Output<Tensor>(framework::GradVarName("Y"));
     int axis = ctx.Attr<int>("axis");

paddle/fluid/operators/lod_tensor_to_array_op.cc

@@ -11,10 +11,13 @@ distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License. */
+#include <algorithm>
+#include <map>
 #include "paddle/fluid/framework/lod_rank_table.h"
 #include "paddle/fluid/framework/lod_tensor_array.h"
 #include "paddle/fluid/framework/op_registry.h"
 #include "paddle/fluid/operators/detail/safe_ref.h"
+#include "paddle/fluid/operators/math/concat.h"
 #include "paddle/fluid/platform/device_context.h"
 #include "paddle/fluid/platform/port.h"
 
@@ -26,6 +29,61 @@ struct CopyRange {
   size_t end;
 };
 
+struct LoDTensorToArrayFunctor;
+
+template <typename DeviceContext>
+struct LoDTensorToArrayFunctorImpl {
+  const LoDTensorToArrayFunctor *prev_functor_;
+  DeviceContext *dev_ctx_;
+  template <typename T>
+  void apply();
+};
+
+struct LoDTensorToArrayFunctor : public boost::static_visitor<void> {
+  std::vector<const framework::Tensor *> ref_inputs_;
+  mutable std::vector<framework::Tensor *> outputs_;
+  const framework::Tensor &input_;
+
+  explicit LoDTensorToArrayFunctor(const framework::Tensor &input)
+      : input_(input) {}
+
+  void AddOutput(framework::Tensor *t) {
+    outputs_.emplace_back(t);
+    ref_inputs_.emplace_back(t);
+  }
+
+  template <typename Place>
+  void operator()(Place place) const {
+    auto &pool = platform::DeviceContextPool::Instance();
+    auto *dev_ctx = pool.Get(place);
+    if (std::is_same<Place, platform::CPUPlace>::value) {
+      Apply(static_cast<platform::CPUDeviceContext *>(dev_ctx));
+    } else {
+#ifdef PADDLE_WITH_CUDA
+      Apply(static_cast<platform::CUDADeviceContext *>(dev_ctx));
+#else
+      PADDLE_THROW("Not compiled with cuda");
+#endif
+    }
+  }
+
+  template <typename DeviceContext>
+  void Apply(DeviceContext *dev_ctx) const {
+    LoDTensorToArrayFunctorImpl<DeviceContext> func;
+    func.prev_functor_ = this;
+    func.dev_ctx_ = dev_ctx;
+    framework::VisitDataType(framework::ToDataType(input_.type()), func);
+  }
+};
+
+template <typename DeviceContext>
+template <typename T>
+void LoDTensorToArrayFunctorImpl<DeviceContext>::apply() {
+  math::ConcatGradFunctor<DeviceContext, T> func;
+  func(*dev_ctx_, prev_functor_->input_, prev_functor_->ref_inputs_, 0,
+       &prev_functor_->outputs_);
+}
+
 class LoDTensorToArrayOp : public framework::OperatorBase {
  public:
   LoDTensorToArrayOp(const std::string &type,
@@ -72,6 +130,11 @@ class LoDTensorToArrayOp : public framework::OperatorBase {
         copy_ranges[t].emplace_back(CopyRange{start_offset, end_offset});
       }
     }
+
+    auto &outputs = *const_cast<framework::Scope &>(scope)
+                         .Var()
+                         ->GetMutable<std::map<size_t, framework::Tensor>>();
+
     for (size_t i = 0; i < max_seq_len; ++i) {
       auto &ranges = copy_ranges[i];
       size_t height = std::accumulate(
@@ -90,17 +153,16 @@ class LoDTensorToArrayOp : public framework::OperatorBase {
         // out[i][offset: offset+len] = x[each_range.begin: each_range.end]
         auto slice = out[i].Slice(static_cast<int>(offset),
                                   static_cast<int>(offset + len));
-
+        outputs.insert({each_range.begin, slice});
-        platform::DeviceContextPool &pool =
-            platform::DeviceContextPool::Instance();
-        auto &dev_ctx = *pool.Get(place);
-
-        framework::TensorCopy(x.Slice(static_cast<int>(each_range.begin),
-                                      static_cast<int>(each_range.end)),
-                              x.place(), dev_ctx, &slice);
         offset += len;
       }
     }
+
+    LoDTensorToArrayFunctor functor(x);
+    for (auto &out_pair : outputs) {
+      functor.AddOutput(&out_pair.second);
+    }
+    platform::VisitPlace(place, functor);
   }
 };
 

paddle/fluid/operators/matmul_op.cc

@@ -59,7 +59,8 @@ class MatMulKernel : public framework::OpKernel<T> {
         RowMatrixFromVector(x.dims()), 0, context.Attr<bool>("transpose_X"));
     auto mat_dim_b = math::CreateMatrixDescriptor(
         ColumnMatrixFromVector(y.dims()), 0, context.Attr<bool>("transpose_Y"));
-    blas.MatMul(x, mat_dim_a, y, mat_dim_b, T(1), out, T(0));
+    auto scale = static_cast<T>(context.Attr<float>("alpha"));
+    blas.MatMul(x, mat_dim_a, y, mat_dim_b, scale, out, T(0));
   }
 };
 
@@ -185,7 +186,8 @@ class MatMulGradKernel : public framework::OpKernel<T> {
     auto blas = math::GetBlas<DeviceContext, T>(context);
     auto mat_dim_a = math::CreateMatrixDescriptor(a.dims(), 0, trans_a);
     auto mat_dim_b = math::CreateMatrixDescriptor(b.dims(), 0, trans_b);
-    blas.MatMul(a, mat_dim_a, b, mat_dim_b, T(1), out, T(0));
+    blas.MatMul(a, mat_dim_a, b, mat_dim_b,
+                static_cast<T>(context.Attr<float>("alpha")), out, T(0));
   }
 
   void CalcInputGrad(const framework::ExecutionContext &context,
@@ -334,6 +336,7 @@ class MatMulOpMaker : public framework::OpProtoAndCheckerMaker {
                   R"DOC(If true, use the transpose of `Y`.
         )DOC")
         .SetDefault(false);
+    AddAttr<float>("alpha", "The scale of Out").SetDefault(1.0f);
     AddComment(R"DOC(
 MatMul Operator.
 

paddle/fluid/operators/mul_op.cc

@@ -156,12 +156,29 @@ class MulGradOp : public framework::OperatorWithKernel {
   }
 };
 
+class MulOpGradMaker : public framework::SingleGradOpDescMaker {
+ public:
+  using framework::SingleGradOpDescMaker::SingleGradOpDescMaker;
+
+ protected:
+  std::unique_ptr<framework::OpDesc> Apply() const override {
+    std::unique_ptr<framework::OpDesc> retv(new framework::OpDesc());
+    retv->SetType("mul_grad");
+    retv->SetInput("X", Input("X"));
+    retv->SetInput("Y", Input("Y"));
+    retv->SetInput(framework::GradVarName("Out"), OutputGrad("Out"));
+    retv->SetOutput(framework::GradVarName("X"), InputGrad("X"));
+    retv->SetOutput(framework::GradVarName("Y"), InputGrad("Y"));
+    retv->SetAttrMap(Attrs());
+    return retv;
+  }
+};
+
 }  // namespace operators
 }  // namespace paddle
 
 namespace ops = paddle::operators;
-REGISTER_OPERATOR(mul, ops::MulOp, ops::MulOpMaker,
+REGISTER_OPERATOR(mul, ops::MulOp, ops::MulOpMaker, ops::MulOpGradMaker);
-                  paddle::framework::DefaultGradOpDescMaker<true>);
 REGISTER_OPERATOR(mul_grad, ops::MulGradOp);
 REGISTER_OP_CPU_KERNEL(
     mul, ops::MulKernel<paddle::platform::CPUDeviceContext, float>,

paddle/fluid/operators/scale_op.cc

@@ -52,6 +52,12 @@ $$Out = scale*X$$
 )DOC");
     AddAttr<float>("scale", "The scaling factor of the scale operator.")
         .SetDefault(1.0);
+    AddAttr<float>("bias", "The bias of the scale operator.").SetDefault(0.0);
+    AddAttr<bool>(
+        "bias_after_scale",
+        "Apply bias addition after or before scaling. It is useful for "
+        "numeric stability in some circumstances.")
+        .SetDefault(true);
   }
 };
 
@@ -80,6 +86,8 @@ class ScaleGradMaker : public framework::SingleGradOpDescMaker {
     grad_op->SetInput("X", OutputGrad("Out"));
     grad_op->SetOutput("Out", InputGrad("X"));
     grad_op->SetAttr("scale", GetAttr("scale"));
+    grad_op->SetAttr("bias", 0.0f);
+    grad_op->SetAttr("bias_after_scale", true);
     return std::unique_ptr<framework::OpDesc>(grad_op);
   }
 };

paddle/fluid/operators/scale_op.h

@@ -34,6 +34,8 @@ class ScaleKernel : public framework::OpKernel<T> {
                       "in and out should have the same dim");
 
     auto scale = static_cast<T>(ctx.Attr<float>("scale"));
+    auto bias = static_cast<T>(ctx.Attr<float>("bias"));
+    auto bias_after_scale = ctx.Attr<bool>("bias_after_scale");
 
     if (in_var->IsType<framework::SelectedRows>() && in_var != out_var) {
       auto& in_slr = in_var->Get<framework::SelectedRows>();
@@ -45,7 +47,11 @@ class ScaleKernel : public framework::OpKernel<T> {
     auto eigen_out = framework::EigenVector<T>::Flatten(*out);
     auto eigen_in = framework::EigenVector<T>::Flatten(*in);
     auto& dev = *ctx.template device_context<DeviceContext>().eigen_device();
-    eigen_out.device(dev) = scale * eigen_in;
+    if (bias_after_scale) {
+      eigen_out.device(dev) = scale * eigen_in + bias;
+    } else {
+      eigen_out.device(dev) = scale * (eigen_in + bias);
+    }
   }
 };
 

paddle/fluid/operators/truncated_gaussian_random_op.cu

@@ -0,0 +1,76 @@
+/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <thrust/random.h>
+#include <thrust/transform.h>
+#include <limits>
+#include "paddle/fluid/framework/op_registry.h"
+#include "paddle/fluid/framework/operator.h"
+
+namespace paddle {
+namespace operators {
+
+template <typename T>
+struct TruncatedNormal {
+  T mean, std;
+  T a_normal_cdf;
+  T b_normal_cdf;
+  unsigned int seed;
+  T numeric_min;
+
+  __host__ __device__ TruncatedNormal(T mean, T std, T numeric_min, int seed)
+      : mean(mean), std(std), seed(seed), numeric_min(numeric_min) {
+    a_normal_cdf = (1.0 + erff(-2.0 / sqrtf(2.0))) / 2.0;
+    b_normal_cdf = (1.0 + erff(2.0 / sqrtf(2.0))) / 2.0;
+  }
+
+  __host__ __device__ T operator()(const unsigned int n) const {
+    thrust::minstd_rand rng;
+    rng.seed(seed);
+    thrust::uniform_real_distribution<T> dist(numeric_min, 1);
+    rng.discard(n);
+    T value = dist(rng);
+    auto p = a_normal_cdf + (b_normal_cdf - a_normal_cdf) * value;
+    return (std::sqrt(2.0) * erfinvf(2 * p - 1) + mean) * std;
+  }
+};
+
+template <typename T>
+class GPUTruncatedGaussianRandomKernel : public framework::OpKernel<T> {
+ public:
+  void Compute(const framework::ExecutionContext& context) const override {
+    auto* tensor = context.Output<framework::Tensor>("Out");
+    T* data = tensor->mutable_data<T>(context.GetPlace());
+    unsigned int seed = static_cast<unsigned int>(context.Attr<int>("seed"));
+    if (seed == 0) {
+      std::random_device rd;
+      seed = rd();
+    }
+    T mean = static_cast<T>(context.Attr<float>("mean"));
+    T std = static_cast<T>(context.Attr<float>("std"));
+    thrust::counting_iterator<unsigned int> index_sequence_begin(0);
+    int64_t size = tensor->numel();
+    thrust::transform(
+        index_sequence_begin, index_sequence_begin + size,
+        thrust::device_ptr<T>(data),
+        TruncatedNormal<T>(mean, std, std::numeric_limits<T>::min(), seed));
+  }
+};
+
+}  // namespace operators
+}  // namespace paddle
+
+REGISTER_OP_CUDA_KERNEL(
+    truncated_gaussian_random,
+    paddle::operators::GPUTruncatedGaussianRandomKernel<float>);

paddle/fluid/operators/while_op.cc

@@ -1,16 +1,16 @@
-/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
+// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
-
+//
-Licensed under the Apache License, Version 2.0 (the "License");
+// Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
+// you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
+// You may obtain a copy of the License at
-
+//
-    http://www.apache.org/licenses/LICENSE-2.0
+//     http://www.apache.org/licenses/LICENSE-2.0
-
+//
-Unless required by applicable law or agreed to in writing, software
+// Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
+// distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
+// See the License for the specific language governing permissions and
-limitations under the License. */
+// limitations under the License.
 
 #include <vector>
 #include "paddle/fluid/framework/executor.h"
@@ -138,6 +138,10 @@ class WhileGradOp : public framework::OperatorBase {
         auto inside_og_name = inside_og_names[i];
         VLOG(8) << "Linking outside " << outside_og_name << " --> inside "
                 << inside_og_name;
+        if (scope.FindVar(outside_og_name) == nullptr) {
+          continue;
+        }
+
         auto &og_outside =
             detail::Ref(scope.FindVar(outside_og_name),
                         "Cannot find Outside Gradient %s", outside_og_name);
@@ -167,20 +171,46 @@ class WhileGradOp : public framework::OperatorBase {
               PADDLE_ENFORCE_EQ(inside_array[j].numel(), 0);
             }
           }
+        } else {
+          PADDLE_THROW("Currently only support LoDTensor and LoDTensorArray.");
         }
       }
       executor.RunPreparedContext(ctx.get(), *cur_scope_iter, false, true,
                                   true);
 
-      auto &pg_names = Outputs(kXGRAD);
+      // The Outputs(kXGRAD) contains the names of the gradient of parameters
+      // and inputs.
+      auto &pg_ig_names = Outputs(kXGRAD);
       auto &p_names = Inputs(kX);
-      PADDLE_ENFORCE_EQ(pg_names.size(), p_names.size());
+      PADDLE_ENFORCE_EQ(pg_ig_names.size(), p_names.size());
-      for (size_t param_id = 0; param_id < pg_names.size(); ++param_id) {
+      for (size_t param_id = 0; param_id < pg_ig_names.size(); ++param_id) {
-        if (pg_names[param_id] == framework::kEmptyVarName) {
+        if (pg_ig_names[param_id] == framework::kEmptyVarName) {
           continue;  // parameter doesn't have gradient
         }
         auto inside_grad_name = framework::GradVarName(p_names[param_id]);
 
+        // for some grad_op, their input doesn't have gradient,
+        // for example lookup_table_grad_op, the input(Idx) doesn't have
+        // gradient.
+        auto pg_ig_var = cur_scope.FindVar(inside_grad_name);
+        PADDLE_ENFORCE(pg_ig_var != nullptr);
+        if (pg_ig_var->IsType<framework::LoDTensorArray>()) {
+          auto pg_ig_lod_t_arr =
+              pg_ig_var->GetMutable<framework::LoDTensorArray>();
+          bool empty = true;
+          for (auto &each : *pg_ig_lod_t_arr) {
+            if (each.numel() != 0) {
+              empty = false;
+              break;
+            }
+          }
+          if (empty) {
+            LOG(WARNING) << pg_ig_names[param_id]
+                         << " is not found in cur_scope.";
+            continue;
+          }
+        }
+
         //  // TODO(tonyyang-svail): Not sure we need the following
         //  // If does not compute gradient of that variable inside rnn,
         //  just
@@ -194,6 +224,11 @@ class WhileGradOp : public framework::OperatorBase {
         if (cur_scope_iter == step_scopes->rbegin()) {
           auto *var = (*cur_scope_iter)->FindVar(inside_grad_name);
           PADDLE_ENFORCE_NOT_NULL(var, "Can not find var %s", inside_grad_name);
+          PADDLE_ENFORCE(var->IsType<framework::LoDTensorArray>() ||
+                             var->IsType<LoDTensor>(),
+                         "Currently the type of var only can be LoDTensorArray "
+                         "or LoDTensor.");
+
           if (var->IsType<LoDTensor>()) {
             auto &inside_tensor = var->Get<framework::LoDTensor>();
             framework::AttributeMap attrs;
@@ -201,7 +236,7 @@ class WhileGradOp : public framework::OperatorBase {
             attrs["shape"] = framework::vectorize2int(inside_tensor.dims());
             attrs["value"] = 0.0f;
 
-            auto var_name = pg_names[param_id];
+            auto var_name = pg_ig_names[param_id];
             auto zero_op = framework::OpRegistry::CreateOp(
                 "fill_constant", framework::VariableNameMap{},
                 {{"Out", {var_name}}}, attrs);
@@ -213,8 +248,8 @@ class WhileGradOp : public framework::OperatorBase {
         }
         auto new_inside_name = cur_scope.Rename(inside_grad_name);
         auto sum_op = framework::OpRegistry::CreateOp(
-            "sum", {{"X", {pg_names[param_id], new_inside_name}}},
+            "sum", {{"X", {pg_ig_names[param_id], new_inside_name}}},
-            {{"Out", {pg_names[param_id]}}},
+            {{"Out", {pg_ig_names[param_id]}}},
             framework::AttributeMap{{"use_mkldnn", {false}}});
         sum_op->Run(cur_scope, dev_place);
         cur_scope.Rename(new_inside_name, inside_grad_name);
@@ -281,6 +316,7 @@ class WhileGradOpDescMaker : public framework::SingleGradOpDescMaker {
              parent_block->FindVarRecursive(input_name) != nullptr)) {
           continue;
         }
+
         output_grads.insert(input_name);
       }
       for (auto &output_name : op->OutputArgumentNames()) {
@@ -309,13 +345,13 @@ class WhileGradOpVarTypeInference : public framework::VarTypeInference {
   void operator()(const framework::OpDesc &op_desc,
                   framework::BlockDesc *block) const override {
     auto p_names = op_desc.Input(kX);
-    auto pg_names = op_desc.Output(framework::GradVarName(kX));
+    auto pg_ig_names = op_desc.Output(framework::GradVarName(kX));
 
     for (size_t i = 0; i < p_names.size(); ++i) {
       auto &p_var = detail::Ref(block->FindVarRecursive(p_names[i]));
-      auto *g_var = block->FindVarRecursive(pg_names[i]);
+      auto *g_var = block->FindVarRecursive(pg_ig_names[i]);
       if (g_var != nullptr) {  // Gradient could be @EMPTY@
-        VLOG(5) << "Setting " << pg_names[i] << " following " << p_names[i]
+        VLOG(5) << "Setting " << pg_ig_names[i] << " following " << p_names[i]
                 << " type: " << p_var.GetType();
         g_var->SetType(p_var.GetType());
         g_var->SetDataType(p_var.GetDataType());
@@ -333,21 +369,21 @@ class WhileGradOpShapeInference : public framework::InferShapeBase {
     ctx->HasInputs(framework::GradVarName(kOutputs));
 
     auto p_names = ctx->Inputs(kX);
-    auto pg_names = ctx->Outputs(kXGRAD);
+    auto pg_ig_names = ctx->Outputs(kXGRAD);
     auto var_types = ctx->GetInputsVarType(kX);
     std::vector<std::string> names_to_set;
     std::vector<framework::DDim> dims_to_set;
     for (size_t i = 0; i < p_names.size(); ++i) {
-      if (pg_names[i] == framework::kEmptyVarName) {
+      if (pg_ig_names[i] == framework::kEmptyVarName) {
         continue;
       }
       auto dims = ctx->GetInputsElementDim(kX, i);
       if (var_types[i] == framework::proto::VarType::LOD_TENSOR) {
-        names_to_set.push_back(pg_names[i]);
+        names_to_set.push_back(pg_ig_names[i]);
         dims_to_set.push_back(dims);
       } else if (var_types[i] == framework::proto::VarType::LOD_TENSOR_ARRAY) {
         // not sure how to set the dim of LOD_TENSOR_ARRAY
-        names_to_set.push_back(pg_names[i]);
+        names_to_set.push_back(pg_ig_names[i]);
         dims_to_set.push_back(dims);
       }
     }

paddle/fluid/pybind/pybind.cc

@@ -396,11 +396,6 @@ All parameter, weight, gradient are variables in Paddle.
     Prune(*prog_with_targets.Proto(), &pruned_desc);
     return new ProgramDesc(pruned_desc);
   });
-  m.def("inference_optimize", [](ProgramDesc &origin) {
-    proto::ProgramDesc pruned_desc;
-    InferenceOptimize(*(origin.Proto()), &pruned_desc);
-    return new ProgramDesc(pruned_desc);
-  });
   m.def("empty_var_name",
         []() { return std::string(framework::kEmptyVarName); });
   m.def("grad_var_suffix",

python/paddle/fluid/backward.py

@@ -347,7 +347,7 @@ def _append_backward_ops_(block,
         # If the op has its own sub-block, deal with the sub-block first
         if op.has_attr("sub_block"):
             sub_block = program.block(op.block_attr_id("sub_block"))
-            grad_sub_block = program.create_block()
+            grad_sub_block = program._create_block()
             grad_sub_block._set_forward_block_idx(sub_block.idx)
             cb = _callback_lookup_(op)
             if cb is not None:
@@ -361,7 +361,7 @@ def _append_backward_ops_(block,
                 _append_backward_ops_(sub_block, sub_block.ops, grad_sub_block,
                                       no_grad_dict, grad_to_var, callbacks)
 
-            program.rollback()
+            program._rollback()
             grad_sub_block_list.append(grad_sub_block.desc)
 
         # Getting op's corresponding grad_op

python/paddle/fluid/clip.py

@@ -331,7 +331,7 @@ def append_gradient_clip_ops(param_grads):
     for p, g in param_grads:
         if g is None:
             continue
-        with p.block.program.optimized_guard([p, g]):
+        with p.block.program._optimized_guard([p, g]):
             clip_attr = getattr(p, 'gradient_clip_attr', NullGradientClipAttr())
             if clip_attr is None:
                 clip_attr = NullGradientClipAttr()
@@ -346,7 +346,7 @@ def append_gradient_clip_ops(param_grads):
     for p, g in param_grads:
         if g is None:
             continue
-        with p.block.program.optimized_guard([p, g]):
+        with p.block.program._optimized_guard([p, g]):
             res.append(clip_attr._create_operators(param=p, grad=g))
 
     return res

python/paddle/fluid/concurrency.py

@@ -126,7 +126,7 @@ class SelectCase(object):
         self.channel = channel
 
     def __enter__(self):
-        self.block = self.main_program.create_block()
+        self.block = self.main_program._create_block()
 
     def construct_op(self):
         main_program = self.helper.main_program
@@ -187,7 +187,7 @@ class SelectCase(object):
                                 if self.value else '')
 
     def __exit__(self, exc_type, exc_val, exc_tb):
-        self.main_program.rollback()
+        self.main_program._rollback()
         if exc_type is not None:
             return False  # re-raise exception
         return True