update layers used in transformer dygraph model, test=develop (#22051)

* update layers, test=develop

* update layers for resnet, test=develop

* fix is_test attr, test=develop

* update cycle_gan, test=develop

* update reinforcement_learning, test=develop

* update ocr, test=develop

* fix bug, test=develop

python/paddle/fluid/dygraph/nn.py

@@ -19,7 +19,7 @@ from .. import core
 from ..layers import utils
 from ..dygraph import dygraph_utils
 from . import layers
-from ..framework import Variable, in_dygraph_mode, OpProtoHolder, Parameter
+from ..framework import Variable, in_dygraph_mode, OpProtoHolder, Parameter, _dygraph_tracer, _varbase_creator
 from ..param_attr import ParamAttr
 from ..initializer import Normal, Constant, NumpyArrayInitializer
 from .. import unique_name
@@ -1134,8 +1134,35 @@ class BatchNorm(layers.Layer):
         # mean and mean_out share the same memory
         mean_out = self._mean
         # variance and variance out share the same memory
+
         variance_out = self._variance
+        attrs = {
+            "momentum": self._momentum,
+            "epsilon": self._epsilon,
+            "is_test": self._is_test,
+            "data_layout": self._data_layout,
+            "use_mkldnn": False,
+            "fuse_with_relu": self._fuse_with_relu,
+            "use_global_stats": self._use_global_stats,
+            "trainable_statistics": self._trainable_statistics
+        }
 
+        inputs = {
+            "X": [input],
+            "Scale": [self.weight],
+            "Bias": [self.bias],
+            "Mean": [self._mean],
+            "Variance": [self._variance]
+        }
+
+        if in_dygraph_mode():
+            attrs['is_test'] = not _dygraph_tracer()._train_mode
+            saved_mean = _varbase_creator(dtype=self._dtype)
+            saved_variance = _varbase_creator(dtype=self._dtype)
+            batch_norm_out = _varbase_creator(dtype=self._dtype)
+            batch_norm_out.stop_gradient = False
+            # inplace is not supported currently
+        else:
             saved_mean = self._helper.create_variable_for_type_inference(
                 dtype=self._dtype, stop_gradient=True)
             saved_variance = self._helper.create_variable_for_type_inference(
@@ -1143,32 +1170,21 @@ class BatchNorm(layers.Layer):
             batch_norm_out = input if self._in_place else self._helper.create_variable_for_type_inference(
                 self._dtype)
 
+        outputs = {
+            "Y": [batch_norm_out],
+            "MeanOut": [mean_out],
+            "VarianceOut": [variance_out],
+            "SavedMean": [saved_mean],
+            "SavedVariance": [saved_variance]
+        }
+
+        if in_dygraph_mode():
+            outs = core.ops.batch_norm(inputs, attrs, outputs)
+            return dygraph_utils._append_activation_in_dygraph(
+                batch_norm_out, act=self._act)
+
         self._helper.append_op(
-            type="batch_norm",
-            inputs={
-                "X": input,
-                "Scale": self.weight,
-                "Bias": self.bias,
-                "Mean": self._mean,
-                "Variance": self._variance
-            },
-            outputs={
-                "Y": batch_norm_out,
-                "MeanOut": mean_out,
-                "VarianceOut": variance_out,
-                "SavedMean": saved_mean,
-                "SavedVariance": saved_variance
-            },
-            attrs={
-                "momentum": self._momentum,
-                "epsilon": self._epsilon,
-                "is_test": self._is_test,
-                "data_layout": self._data_layout,
-                "use_mkldnn": False,
-                "fuse_with_relu": self._fuse_with_relu,
-                "use_global_stats": self._use_global_stats,
-                "trainable_statistics": self._trainable_statistics
-            })
+            type="batch_norm", inputs=inputs, outputs=outputs, attrs=attrs)
 
         # Currently, we don't support inplace in dygraph mode
         return self._helper.append_activation(batch_norm_out, self._act)
@@ -1454,11 +1470,23 @@ class LayerNorm(layers.Layer):
                 ', expected input with shape [*, ' + str_normalized_shape[
                     1:] + ', but got input shape ' + str(input_shape))
         inputs = dict()
-        inputs['X'] = input
+        inputs['X'] = [input]
         if self._scale:
-            inputs['Scale'] = self.weight
+            inputs['Scale'] = [self.weight]
         if self._shift:
-            inputs['Bias'] = self.bias
+            inputs['Bias'] = [self.bias]
+
+        attrs = {
+            "epsilon": self._epsilon,
+            "begin_norm_axis": self._begin_norm_axis
+        }
+
+        if in_dygraph_mode():
+            outs = core.ops.layer_norm(inputs, attrs)
+            pre_act = outs['Y'][0]
+            return dygraph_utils._append_activation_in_dygraph(
+                pre_act, act=self._act)
+
         # create output
         mean_out = self._helper.create_variable_for_type_inference(
             dtype=self._dtype, stop_gradient=True)
@@ -1623,9 +1651,22 @@ class GRUUnit(layers.Layer):
             attr=bias_attr, shape=bias_size, dtype=dtype, is_bias=True)
 
     def forward(self, input, hidden):
-        inputs = {'Input': input, 'HiddenPrev': hidden, 'Weight': self.weight}
+        inputs = {
+            'Input': [input],
+            'HiddenPrev': [hidden],
+            'Weight': [self.weight]
+        }
         if self.bias:
-            inputs['Bias'] = self.bias
+            inputs['Bias'] = [self.bias]
+        attrs = {
+            'activation': self.activation,
+            'gate_activation': self.gate_activation,
+        }
+
+        if in_dygraph_mode():
+            outs = core.ops.gru_unit(inputs, attrs)
+            return outs['Hidden'][0], outs['ResetHiddenPrev'][0], outs['Gate'][
+                0]
 
         gate = self._helper.create_variable_for_type_inference(self._dtype)
         reset_hidden_pre = self._helper.create_variable_for_type_inference(
@@ -2277,21 +2318,32 @@ class Conv2DTranspose(layers.Layer):
             is_bias=True)
 
     def forward(self, input):
-        pre_bias = self._helper.create_variable_for_type_inference(
-            dtype=input.dtype)
-        self._helper.append_op(
-            type=self._op_type,
-            inputs={'Input': [input],
-                    'Filter': [self.weight]},
-            outputs={'Output': pre_bias},
-            attrs={
+        inputs = {'Input': [input], 'Filter': [self.weight]}
+        attrs = {
             'output_size': self._output_size,
             'strides': self._stride,
             'paddings': self._padding,
             'dilations': self._dilation,
             'groups': self._groups,
             'use_cudnn': self._use_cudnn
-            })
+        }
+
+        if in_dygraph_mode():
+            op = getattr(core.ops, self._op_type)
+            outs = op(inputs, attrs)
+            pre_bias = outs['Output'][0]
+            pre_act = dygraph_utils._append_bias_in_dygraph(pre_bias, self.bias,
+                                                            1)
+            return dygraph_utils._append_activation_in_dygraph(
+                pre_act, act=self._act)
+
+        pre_bias = self._helper.create_variable_for_type_inference(
+            dtype=input.dtype)
+        self._helper.append_op(
+            type=self._op_type,
+            inputs=inputs,
+            outputs={'Output': pre_bias},
+            attrs=attrs)
 
         if self.bias is not None:
             pre_act = self._helper.create_variable_for_type_inference(

python/paddle/fluid/layers/tensor.py

@@ -16,7 +16,7 @@ from __future__ import print_function
 from six.moves import reduce
 from ..layer_helper import LayerHelper
 from ..param_attr import ParamAttr
-from ..framework import convert_np_dtype_to_dtype_, in_dygraph_mode
+from ..framework import convert_np_dtype_to_dtype_, in_dygraph_mode, _varbase_creator
 from ..framework import Variable
 from ..initializer import Constant, force_init_on_cpu
 from ..core import VarDesc
@@ -552,6 +552,43 @@ def fill_constant(shape, dtype, value, force_cpu=False, out=None):
           shape = fluid.layers.fill_constant([1,2], "int32", 2) # shape=[2,2]
           data4 = fluid.layers.fill_constant(shape=shape, dtype='bool', value=True) # data4=[[True,True],[True,True]]
     """
+
+    def _contain_var(one_list):
+        for ele in one_list:
+            if isinstance(ele, Variable):
+                return True
+        return False
+
+    attrs = {
+        'value': float(value),
+        'force_cpu': force_cpu or force_init_on_cpu()
+    }
+
+    if convert_dtype(dtype) in ['int64', 'int32']:
+        attrs['str_value'] = str(int(value))
+    else:
+        attrs['str_value'] = str(float(value))
+
+    if in_dygraph_mode():
+        if isinstance(shape, (list, tuple)):
+            contain_var = _contain_var(shape)
+            if contain_var:
+                raise TypeError(
+                    "The type of 'shape' in fill_constant must be list[int] or tuple(int) in Dygraph mode, but "
+                    "received %s, which contains Variable." % type(shape))
+            attrs['shape'] = shape
+        else:
+            raise TypeError(
+                "The type of 'shape' in fill_constant must be list[int] or tuple(int) in Dygraph mode, but "
+                "received %s." % type(shape))
+        if out is None:
+            out = _varbase_creator(dtype=dtype)
+        attrs['dtype'] = out.dtype
+        outputs = {'Out': [out]}
+        outs = core.ops.fill_constant({}, attrs, outputs)
+        out.stop_gradient = True
+        return out
+
     helper = LayerHelper("fill_constant", **locals())
     check_dtype(dtype, 'create data type',
                 ['bool', 'float16', 'float32', 'float64', 'int32', 'int64'],
@@ -568,12 +605,6 @@ def fill_constant(shape, dtype, value, force_cpu=False, out=None):
     else:
         attrs['str_value'] = str(float(value))
 
-    def _contain_var(one_list):
-        for ele in one_list:
-            if isinstance(ele, Variable):
-                return True
-        return False
-
     def _get_attr_shape(list_shape):
         attr_shape = []
         for idx, dim in enumerate(list_shape):

python/paddle/fluid/optimizer.py

@@ -896,21 +896,30 @@ class MomentumOptimizer(Optimizer):
 
         velocity_acc = self._get_accumulator(self._velocity_acc_str,
                                              param_and_grad[0])
+        attrs = {"mu": self._momentum, "use_nesterov": self._use_nesterov}
+
+        inputs = {
+            "Param": [param_and_grad[0]],
+            "Grad": [param_and_grad[1]],
+            "Velocity": [velocity_acc],
+            "LearningRate": [self._create_param_lr(param_and_grad)]
+        }
+
+        outputs = {
+            "ParamOut": [param_and_grad[0]],
+            "VelocityOut": [velocity_acc]
+        }
+
+        if framework.in_dygraph_mode():
+            core.ops.momentum(inputs, attrs, outputs)
+            return None
+
         # create the momentum optimize op
         momentum_op = block.append_op(
             type=self.type,
-            inputs={
-                "Param": param_and_grad[0],
-                "Grad": param_and_grad[1],
-                "Velocity": velocity_acc,
-                "LearningRate": self._create_param_lr(param_and_grad)
-            },
-            outputs={
-                "ParamOut": param_and_grad[0],
-                "VelocityOut": velocity_acc
-            },
-            attrs={"mu": self._momentum,
-                   "use_nesterov": self._use_nesterov},
+            inputs=inputs,
+            outputs=outputs,
+            attrs=attrs,
             stop_gradient=True)
 
         return momentum_op

python/paddle/fluid/regularizer.py

@@ -15,6 +15,7 @@
 from __future__ import print_function
 
 from . import framework
+from .framework import in_dygraph_mode, _varbase_creator
 from . import core
 
 __all__ = ['L1Decay', 'L2Decay', 'L1DecayRegularizer', 'L2DecayRegularizer']
@@ -74,10 +75,12 @@ def append_regularization_ops(parameters_and_grads, regularization=None):
                     lod_level=param.lod_level,
                     type=core.VarDesc.VarType.LOD_TENSOR)
 
-            grad.block.append_op(
-                type='sum',
-                inputs={"X": [grad, regularization_term]},
-                outputs={"Out": new_grad})
+            inputs = {"X": [grad, regularization_term]}
+            outputs = {"Out": [new_grad]}
+            if in_dygraph_mode():
+                core.ops.sum(inputs, {}, outputs)
+            else:
+                grad.block.append_op(type='sum', inputs=inputs, outputs=outputs)
 
             params_and_grads.append((param, new_grad))
 
@@ -165,8 +168,12 @@ class L2DecayRegularizer(WeightDecayRegularizer):
         assert isinstance(param, framework.Parameter)
         assert isinstance(block, framework.Block)
 
+        inputs = {"X": [param]}
+        attrs = {"scale": self._regularization_coeff}
+
         if framework.in_dygraph_mode():
-            decay = block.create_var(dtype=param.dtype, shape=param.shape)
+            outs = core.ops.scale(inputs, attrs)
+            return outs['Out'][0]
         else:
             decay = block.create_var(
                 dtype=param.dtype, shape=param.shape, lod_level=param.lod_level)

python/paddle/fluid/tests/unittests/test_imperative_resnet.py

@@ -27,6 +27,8 @@ from test_imperative_base import new_program_scope
 from utils import DyGraphProgramDescTracerTestHelper, is_equal_program
 from paddle.fluid.dygraph import TracedLayer
 
+#NOTE(zhiqiu): run with FLAGS_cudnn_deterministic=1
+
 batch_size = 8
 train_parameters = {
     "input_size": [3, 224, 224],