m53297601
/
Paddle
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

update layers used in transformer dygraph model, test=develop (#22051)

Browse Source 
* update layers, test=develop

* update layers for resnet, test=develop

* fix is_test attr, test=develop

* update cycle_gan, test=develop

* update reinforcement_learning, test=develop

* update ocr, test=develop

* fix bug, test=develop
release/1.7
Leo Chen 5 years ago committed by hong
parent 5b2e98aa17
commit 51cb918a05
6 changed files with 285 additions and 128 deletions
Show Stats Download Patch File Download Diff File

146
python/paddle/fluid/dygraph/nn.py
Unescape View File

@ -19,7 +19,7 @@ from .. import core
from ..layers import utils
from ..dygraph import dygraph_utils
from . import layers
from ..framework import Variable, in_dygraph_mode, OpProtoHolder, Parameter
from ..framework import Variable, in_dygraph_mode, OpProtoHolder, Parameter, _dygraph_tracer, _varbase_creator
from ..param_attr import ParamAttr
from ..initializer import Normal, Constant, NumpyArrayInitializer
from .. import unique_name
@ -1134,41 +1134,57 @@ class BatchNorm(layers.Layer):
# mean and mean_out share the same memory
mean_out = self._mean
# variance and variance out share the same memory
variance_out = self._variance
attrs = {
"momentum": self._momentum,
"epsilon": self._epsilon,
"is_test": self._is_test,
"data_layout": self._data_layout,
"use_mkldnn": False,
"fuse_with_relu": self._fuse_with_relu,
"use_global_stats": self._use_global_stats,
"trainable_statistics": self._trainable_statistics
}
saved_mean = self._helper.create_variable_for_type_inference(
dtype=self._dtype, stop_gradient=True)
saved_variance = self._helper.create_variable_for_type_inference(
dtype=self._dtype, stop_gradient=True)
batch_norm_out = input if self._in_place else self._helper.create_variable_for_type_inference(
self._dtype)
inputs = {
"X": [input],
"Scale": [self.weight],
"Bias": [self.bias],
"Mean": [self._mean],
"Variance": [self._variance]
}
if in_dygraph_mode():
attrs['is_test'] = not _dygraph_tracer()._train_mode
saved_mean = _varbase_creator(dtype=self._dtype)
saved_variance = _varbase_creator(dtype=self._dtype)
batch_norm_out = _varbase_creator(dtype=self._dtype)
batch_norm_out.stop_gradient = False
# inplace is not supported currently
else:
saved_mean = self._helper.create_variable_for_type_inference(
dtype=self._dtype, stop_gradient=True)
saved_variance = self._helper.create_variable_for_type_inference(
dtype=self._dtype, stop_gradient=True)
batch_norm_out = input if self._in_place else self._helper.create_variable_for_type_inference(
self._dtype)
outputs = {
"Y": [batch_norm_out],
"MeanOut": [mean_out],
"VarianceOut": [variance_out],
"SavedMean": [saved_mean],
"SavedVariance": [saved_variance]
}
if in_dygraph_mode():
outs = core.ops.batch_norm(inputs, attrs, outputs)
return dygraph_utils._append_activation_in_dygraph(
batch_norm_out, act=self._act)
self._helper.append_op(
type="batch_norm",
inputs={
"X": input,
"Scale": self.weight,
"Bias": self.bias,
"Mean": self._mean,
"Variance": self._variance
},
outputs={
"Y": batch_norm_out,
"MeanOut": mean_out,
"VarianceOut": variance_out,
"SavedMean": saved_mean,
"SavedVariance": saved_variance
},
attrs={
"momentum": self._momentum,
"epsilon": self._epsilon,
"is_test": self._is_test,
"data_layout": self._data_layout,
"use_mkldnn": False,
"fuse_with_relu": self._fuse_with_relu,
"use_global_stats": self._use_global_stats,
"trainable_statistics": self._trainable_statistics
})
type="batch_norm", inputs=inputs, outputs=outputs, attrs=attrs)
# Currently, we don't support inplace in dygraph mode
return self._helper.append_activation(batch_norm_out, self._act)
@ -1454,11 +1470,23 @@ class LayerNorm(layers.Layer):
', expected input with shape [*, ' + str_normalized_shape[
1:] + ', but got input shape ' + str(input_shape))
inputs = dict()
inputs['X'] = input
inputs['X'] = [input]
if self._scale:
inputs['Scale'] = self.weight
inputs['Scale'] = [self.weight]
if self._shift:
inputs['Bias'] = self.bias
inputs['Bias'] = [self.bias]
attrs = {
"epsilon": self._epsilon,
"begin_norm_axis": self._begin_norm_axis
}
if in_dygraph_mode():
outs = core.ops.layer_norm(inputs, attrs)
pre_act = outs['Y'][0]
return dygraph_utils._append_activation_in_dygraph(
pre_act, act=self._act)
# create output
mean_out = self._helper.create_variable_for_type_inference(
dtype=self._dtype, stop_gradient=True)
@ -1623,9 +1651,22 @@ class GRUUnit(layers.Layer):
attr=bias_attr, shape=bias_size, dtype=dtype, is_bias=True)
def forward(self, input, hidden):
inputs = {'Input': input, 'HiddenPrev': hidden, 'Weight': self.weight}
inputs = {
'Input': [input],
'HiddenPrev': [hidden],
'Weight': [self.weight]
}
if self.bias:
inputs['Bias'] = self.bias
inputs['Bias'] = [self.bias]
attrs = {
'activation': self.activation,
'gate_activation': self.gate_activation,
}
if in_dygraph_mode():
outs = core.ops.gru_unit(inputs, attrs)
return outs['Hidden'][0], outs['ResetHiddenPrev'][0], outs['Gate'][
0]
gate = self._helper.create_variable_for_type_inference(self._dtype)
reset_hidden_pre = self._helper.create_variable_for_type_inference(
@ -2277,21 +2318,32 @@ class Conv2DTranspose(layers.Layer):
is_bias=True)
def forward(self, input):
inputs = {'Input': [input], 'Filter': [self.weight]}
attrs = {
'output_size': self._output_size,
'strides': self._stride,
'paddings': self._padding,
'dilations': self._dilation,
'groups': self._groups,
'use_cudnn': self._use_cudnn
}
if in_dygraph_mode():
op = getattr(core.ops, self._op_type)
outs = op(inputs, attrs)
pre_bias = outs['Output'][0]
pre_act = dygraph_utils._append_bias_in_dygraph(pre_bias, self.bias,
1)
return dygraph_utils._append_activation_in_dygraph(
pre_act, act=self._act)
pre_bias = self._helper.create_variable_for_type_inference(
dtype=input.dtype)
self._helper.append_op(
type=self._op_type,
inputs={'Input': [input],
'Filter': [self.weight]},
inputs=inputs,
outputs={'Output': pre_bias},
attrs={
'output_size': self._output_size,
'strides': self._stride,
'paddings': self._padding,
'dilations': self._dilation,
'groups': self._groups,
'use_cudnn': self._use_cudnn
})
attrs=attrs)
if self.bias is not None:
pre_act = self._helper.create_variable_for_type_inference(

156
python/paddle/fluid/layers/nn.py
View File

File diff suppressed because it is too large Load Diff

45
python/paddle/fluid/layers/tensor.py
Unescape View File

@ -16,7 +16,7 @@ from __future__ import print_function
from six.moves import reduce
from ..layer_helper import LayerHelper
from ..param_attr import ParamAttr
from ..framework import convert_np_dtype_to_dtype_, in_dygraph_mode
from ..framework import convert_np_dtype_to_dtype_, in_dygraph_mode, _varbase_creator
from ..framework import Variable
from ..initializer import Constant, force_init_on_cpu
from ..core import VarDesc
@ -552,6 +552,43 @@ def fill_constant(shape, dtype, value, force_cpu=False, out=None):
shape = fluid.layers.fill_constant([1,2], "int32", 2) # shape=[2,2]
data4 = fluid.layers.fill_constant(shape=shape, dtype='bool', value=True) # data4=[[True,True],[True,True]]
"""
def _contain_var(one_list):
for ele in one_list:
if isinstance(ele, Variable):
return True
return False
attrs = {
'value': float(value),
'force_cpu': force_cpu or force_init_on_cpu()
}
if convert_dtype(dtype) in ['int64', 'int32']:
attrs['str_value'] = str(int(value))
else:
attrs['str_value'] = str(float(value))
if in_dygraph_mode():
if isinstance(shape, (list, tuple)):
contain_var = _contain_var(shape)
if contain_var:
raise TypeError(
"The type of 'shape' in fill_constant must be list[int] or tuple(int) in Dygraph mode, but "
"received %s, which contains Variable." % type(shape))
attrs['shape'] = shape
else:
raise TypeError(
"The type of 'shape' in fill_constant must be list[int] or tuple(int) in Dygraph mode, but "
"received %s." % type(shape))
if out is None:
out = _varbase_creator(dtype=dtype)
attrs['dtype'] = out.dtype
outputs = {'Out': [out]}
outs = core.ops.fill_constant({}, attrs, outputs)
out.stop_gradient = True
return out
helper = LayerHelper("fill_constant", **locals())
check_dtype(dtype, 'create data type',
['bool', 'float16', 'float32', 'float64', 'int32', 'int64'],
@ -568,12 +605,6 @@ def fill_constant(shape, dtype, value, force_cpu=False, out=None):
else:
attrs['str_value'] = str(float(value))
def _contain_var(one_list):
for ele in one_list:
if isinstance(ele, Variable):
return True
return False
def _get_attr_shape(list_shape):
attr_shape = []
for idx, dim in enumerate(list_shape):

33
python/paddle/fluid/optimizer.py
Unescape View File

@ -896,21 +896,30 @@ class MomentumOptimizer(Optimizer):
velocity_acc = self._get_accumulator(self._velocity_acc_str,
param_and_grad[0])
attrs = {"mu": self._momentum, "use_nesterov": self._use_nesterov}
inputs = {
"Param": [param_and_grad[0]],
"Grad": [param_and_grad[1]],
"Velocity": [velocity_acc],
"LearningRate": [self._create_param_lr(param_and_grad)]
}
outputs = {
"ParamOut": [param_and_grad[0]],
"VelocityOut": [velocity_acc]
}
if framework.in_dygraph_mode():
core.ops.momentum(inputs, attrs, outputs)
return None
# create the momentum optimize op
momentum_op = block.append_op(
type=self.type,
inputs={
"Param": param_and_grad[0],
"Grad": param_and_grad[1],
"Velocity": velocity_acc,
"LearningRate": self._create_param_lr(param_and_grad)
},
outputs={
"ParamOut": param_and_grad[0],
"VelocityOut": velocity_acc
},
attrs={"mu": self._momentum,
"use_nesterov": self._use_nesterov},
inputs=inputs,
outputs=outputs,
attrs=attrs,
stop_gradient=True)
return momentum_op

31
python/paddle/fluid/regularizer.py
Unescape View File

@ -15,6 +15,7 @@
from __future__ import print_function
from . import framework
from .framework import in_dygraph_mode, _varbase_creator
from . import core
__all__ = ['L1Decay', 'L2Decay', 'L1DecayRegularizer', 'L2DecayRegularizer']
@ -74,10 +75,12 @@ def append_regularization_ops(parameters_and_grads, regularization=None):
lod_level=param.lod_level,
type=core.VarDesc.VarType.LOD_TENSOR)
grad.block.append_op(
type='sum',
inputs={"X": [grad, regularization_term]},
outputs={"Out": new_grad})
inputs = {"X": [grad, regularization_term]}
outputs = {"Out": [new_grad]}
if in_dygraph_mode():
core.ops.sum(inputs, {}, outputs)
else:
grad.block.append_op(type='sum', inputs=inputs, outputs=outputs)
params_and_grads.append((param, new_grad))
@ -165,20 +168,24 @@ class L2DecayRegularizer(WeightDecayRegularizer):
assert isinstance(param, framework.Parameter)
assert isinstance(block, framework.Block)
inputs = {"X": [param]}
attrs = {"scale": self._regularization_coeff}
if framework.in_dygraph_mode():
decay = block.create_var(dtype=param.dtype, shape=param.shape)
outs = core.ops.scale(inputs, attrs)
return outs['Out'][0]
else:
decay = block.create_var(
dtype=param.dtype, shape=param.shape, lod_level=param.lod_level)
# Append Op to calculate decay
block.append_op(
type='scale',
inputs={"X": param},
outputs={"Out": decay},
attrs={"scale": self._regularization_coeff})
# Append Op to calculate decay
block.append_op(
type='scale',
inputs={"X": param},
outputs={"Out": decay},
attrs={"scale": self._regularization_coeff})
return decay
return decay
def __str__(self):
return "L2Decay, regularization_coeff=%f" % self._regularization_coeff

2
python/paddle/fluid/tests/unittests/test_imperative_resnet.py
Unescape View File

@ -27,6 +27,8 @@ from test_imperative_base import new_program_scope
from utils import DyGraphProgramDescTracerTestHelper, is_equal_program
from paddle.fluid.dygraph import TracedLayer
#NOTE(zhiqiu): run with FLAGS_cudnn_deterministic=1
batch_size = 8
train_parameters = {
"input_size": [3, 224, 224],

Write Preview
Loading…
Cancel
Save