From e5fc529159a1b2a80ee883c92a1aca35c943701a Mon Sep 17 00:00:00 2001 From: Wei Luning Date: Tue, 4 Aug 2020 10:51:06 +0800 Subject: [PATCH] z --- mindspore/common/parameter.py | 3 ++- mindspore/nn/cell.py | 4 +++ mindspore/nn/optim/optimizer.py | 5 ++-- model_zoo/official/cv/inceptionv3/train.py | 10 +++++-- .../cv/mobilenetv3/src/mobilenetV3.py | 2 +- model_zoo/official/cv/resnet/train.py | 18 ++++++++----- model_zoo/official/cv/resnet50_quant/train.py | 8 +++--- .../cv/resnext50/src/image_classification.py | 10 +++---- .../cv/resnext50/src/utils/var_init.py | 23 ++++++++-------- .../official/cv/vgg16/src/utils/var_init.py | 23 ++++++++-------- model_zoo/official/cv/vgg16/src/vgg.py | 8 +++--- .../cv/yolov3_darknet53/src/initializer.py | 26 ++++++++++--------- .../official/cv/yolov3_darknet53/src/util.py | 2 +- 13 files changed, 82 insertions(+), 60 deletions(-) diff --git a/mindspore/common/parameter.py b/mindspore/common/parameter.py index d3be0b5e5a..63866e1131 100644 --- a/mindspore/common/parameter.py +++ b/mindspore/common/parameter.py @@ -340,7 +340,8 @@ class Parameter(MetaTensor): Default: False. Returns: - Parameter, Parameter after init data. + Parameter, the `Parameter` after init data. If current `Parameter` already initialized before, + returns the same initialized `Parameter`. """ if self.init_mode is None: return self diff --git a/mindspore/nn/cell.py b/mindspore/nn/cell.py index 4ef7a0a98f..44ef9c3701 100755 --- a/mindspore/nn/cell.py +++ b/mindspore/nn/cell.py @@ -536,6 +536,10 @@ class Cell: """ Init all parameters' data and replace the original saved parameters in cell. + Notes: + trainable_params() and other similar interfaces may return different parameter instance after + `init_parameters_data`, do not save these result. + Args: auto_parallel_mode (bool): If running in auto_parallel_mode. diff --git a/mindspore/nn/optim/optimizer.py b/mindspore/nn/optim/optimizer.py index 00c26dd18f..1fae2fde47 100755 --- a/mindspore/nn/optim/optimizer.py +++ b/mindspore/nn/optim/optimizer.py @@ -425,12 +425,13 @@ class Optimizer(Cell): raise TypeError(f"The parameter only support 'Parameter' or 'list' type.") lr = [] + ids = [id(p) for p in self.parameters] for p in param_list: validator.check_value_type("parameter", p, [Parameter], self.cls_name) - if p not in self.parameters: + if id(p) not in ids: raise ValueError(f"The parameter {p.name} is not in optimizer.") if self.is_group_lr: - index = self.parameters.index(p) + index = ids.index(id(p)) lr.append(get_lr_value(self.learning_rate[index])) else: lr.append(get_lr_value(self.learning_rate)) diff --git a/model_zoo/official/cv/inceptionv3/train.py b/model_zoo/official/cv/inceptionv3/train.py index 518b9f02c4..bca02e6ee0 100644 --- a/model_zoo/official/cv/inceptionv3/train.py +++ b/model_zoo/official/cv/inceptionv3/train.py @@ -84,8 +84,14 @@ if __name__ == '__main__': lr = Tensor(lr) # optimizer - decayed_params = list(filter(lambda x: 'beta' not in x.name and 'gamma' not in x.name and 'bias' not in x.name, net.trainable_params())) - no_decayed_params = [param for param in net.trainable_params() if param not in decayed_params] + decayed_params = [] + no_decayed_params = [] + for param in net.trainable_params(): + if 'beta' not in param.name and 'gamma' not in param.name and 'bias' not in param.name: + decayed_params.append(param) + else: + no_decayed_params.append(param) + group_params = [{'params': decayed_params, 'weight_decay': cfg.weight_decay}, {'params': no_decayed_params}, {'order_params': net.trainable_params()}] diff --git a/model_zoo/official/cv/mobilenetv3/src/mobilenetV3.py b/model_zoo/official/cv/mobilenetv3/src/mobilenetV3.py index 004b5aafcf..b84cbeb83c 100644 --- a/model_zoo/official/cv/mobilenetv3/src/mobilenetV3.py +++ b/model_zoo/official/cv/mobilenetv3/src/mobilenetV3.py @@ -290,7 +290,6 @@ class MobileNetV3(nn.Cell): kernel_size=1, has_bias=True, pad_mode='pad') self.squeeze = P.Squeeze(axis=(2, 3)) - self.init_parameters_data() self._initialize_weights() def construct(self, x): @@ -320,6 +319,7 @@ class MobileNetV3(nn.Cell): Examples: >>> _initialize_weights() """ + self.init_parameters_data() for _, m in self.cells_and_names(): if isinstance(m, (nn.Conv2d)): n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels diff --git a/model_zoo/official/cv/resnet/train.py b/model_zoo/official/cv/resnet/train.py index b721fa0f92..6a9595c4b0 100755 --- a/model_zoo/official/cv/resnet/train.py +++ b/model_zoo/official/cv/resnet/train.py @@ -101,12 +101,12 @@ if __name__ == '__main__': for _, cell in net.cells_and_names(): if isinstance(cell, nn.Conv2d): cell.weight.default_input = weight_init.initializer(weight_init.XavierUniform(), - cell.weight.default_input.shape, - cell.weight.default_input.dtype).to_tensor() + cell.weight.shape, + cell.weight.dtype) if isinstance(cell, nn.Dense): cell.weight.default_input = weight_init.initializer(weight_init.TruncatedNormal(), - cell.weight.default_input.shape, - cell.weight.default_input.dtype).to_tensor() + cell.weight.shape, + cell.weight.dtype) # init lr if args_opt.net == "resnet50": @@ -123,8 +123,14 @@ if __name__ == '__main__': lr = Tensor(lr) # define opt - decayed_params = list(filter(lambda x: 'beta' not in x.name and 'gamma' not in x.name and 'bias' not in x.name, net.trainable_params())) - no_decayed_params = [param for param in net.trainable_params() if param not in decayed_params] + decayed_params = [] + no_decayed_params = [] + for param in net.trainable_params(): + if 'beta' not in param.name and 'gamma' not in param.name and 'bias' not in param.name: + decayed_params.append(param) + else: + no_decayed_params.append(param) + group_params = [{'params': decayed_params, 'weight_decay': config.weight_decay}, {'params': no_decayed_params}, {'order_params': net.trainable_params()}] diff --git a/model_zoo/official/cv/resnet50_quant/train.py b/model_zoo/official/cv/resnet50_quant/train.py index 3d0656b80d..5e1f075615 100755 --- a/model_zoo/official/cv/resnet50_quant/train.py +++ b/model_zoo/official/cv/resnet50_quant/train.py @@ -91,12 +91,12 @@ if __name__ == '__main__': for _, cell in net.cells_and_names(): if isinstance(cell, nn.Conv2d): cell.weight.default_input = weight_init.initializer(weight_init.XavierUniform(), - cell.weight.default_input.shape, - cell.weight.default_input.dtype).to_tensor() + cell.weight.shape, + cell.weight.dtype) if isinstance(cell, nn.Dense): cell.weight.default_input = weight_init.initializer(weight_init.TruncatedNormal(), - cell.weight.default_input.shape, - cell.weight.default_input.dtype).to_tensor() + cell.weight.shape, + cell.weight.dtype) if not config.use_label_smooth: config.label_smooth_factor = 0.0 loss = CrossEntropy(smooth_factor=config.label_smooth_factor, num_classes=config.class_num) diff --git a/model_zoo/official/cv/resnext50/src/image_classification.py b/model_zoo/official/cv/resnext50/src/image_classification.py index de5a4fcd2c..6b12ede367 100644 --- a/model_zoo/official/cv/resnext50/src/image_classification.py +++ b/model_zoo/official/cv/resnext50/src/image_classification.py @@ -63,19 +63,19 @@ class Resnet(ImageClassificationNetwork): if isinstance(cell, nn.Conv2d): cell.weight.default_input = init.initializer( KaimingNormal(a=math.sqrt(5), mode='fan_out', nonlinearity='relu'), - cell.weight.default_input.shape, cell.weight.default_input.dtype).to_tensor() + cell.weight.shape, cell.weight.dtype) elif isinstance(cell, nn.BatchNorm2d): - cell.gamma.default_input = init.initializer('ones', cell.gamma.default_input.shape).to_tensor() - cell.beta.default_input = init.initializer('zeros', cell.beta.default_input.shape).to_tensor() + cell.gamma.default_input = init.initializer('ones', cell.gamma.shape) + cell.beta.default_input = init.initializer('zeros', cell.beta.shape) # Zero-initialize the last BN in each residual branch, # so that the residual branch starts with zeros, and each residual block behaves like an identity. # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677 for cell in self.cells_and_names(): if isinstance(cell, backbones.resnet.Bottleneck): - cell.bn3.gamma.default_input = init.initializer('zeros', cell.bn3.gamma.default_input.shape).to_tensor() + cell.bn3.gamma.default_input = init.initializer('zeros', cell.bn3.gamma.shape) elif isinstance(cell, backbones.resnet.BasicBlock): - cell.bn2.gamma.default_input = init.initializer('zeros', cell.bn2.gamma.default_input.shape).to_tensor() + cell.bn2.gamma.default_input = init.initializer('zeros', cell.bn2.gamma.shape) diff --git a/model_zoo/official/cv/resnext50/src/utils/var_init.py b/model_zoo/official/cv/resnext50/src/utils/var_init.py index 51fc109990..185072d441 100644 --- a/model_zoo/official/cv/resnext50/src/utils/var_init.py +++ b/model_zoo/official/cv/resnext50/src/utils/var_init.py @@ -19,7 +19,6 @@ import math from functools import reduce import numpy as np import mindspore.nn as nn -from mindspore import Tensor from mindspore.common import initializer as init def _calculate_gain(nonlinearity, param=None): @@ -191,23 +190,25 @@ def default_recurisive_init(custom_cell): for _, cell in custom_cell.cells_and_names(): if isinstance(cell, nn.Conv2d): cell.weight.default_input = init.initializer(KaimingUniform(a=math.sqrt(5)), - cell.weight.default_input.shape, - cell.weight.default_input.dtype).to_tensor() + cell.weight.shape, + cell.weight.dtype) if cell.bias is not None: - fan_in, _ = _calculate_in_and_out(cell.weight.default_input.asnumpy()) + fan_in, _ = _calculate_in_and_out(cell.weight) bound = 1 / math.sqrt(fan_in) np.random.seed(0) - cell.bias.default_input = Tensor(np.random.uniform(-bound, bound, cell.bias.default_input.shape), - cell.bias.default_input.dtype) + cell.bias.default_input = init.initializer(init.Uniform(bound), + cell.bias.shape, + cell.bias.dtype) elif isinstance(cell, nn.Dense): cell.weight.default_input = init.initializer(KaimingUniform(a=math.sqrt(5)), - cell.weight.default_input.shape, - cell.weight.default_input.dtype).to_tensor() + cell.weight.shape, + cell.weight.dtype) if cell.bias is not None: - fan_in, _ = _calculate_in_and_out(cell.weight.default_input.asnumpy()) + fan_in, _ = _calculate_in_and_out(cell.weight) bound = 1 / math.sqrt(fan_in) np.random.seed(0) - cell.bias.default_input = Tensor(np.random.uniform(-bound, bound, cell.bias.default_input.shape), - cell.bias.default_input.dtype) + cell.bias.default_input = init.initializer(init.Uniform(bound), + cell.bias.shape, + cell.bias.dtype) elif isinstance(cell, (nn.BatchNorm2d, nn.BatchNorm1d)): pass diff --git a/model_zoo/official/cv/vgg16/src/utils/var_init.py b/model_zoo/official/cv/vgg16/src/utils/var_init.py index 51fc109990..185072d441 100644 --- a/model_zoo/official/cv/vgg16/src/utils/var_init.py +++ b/model_zoo/official/cv/vgg16/src/utils/var_init.py @@ -19,7 +19,6 @@ import math from functools import reduce import numpy as np import mindspore.nn as nn -from mindspore import Tensor from mindspore.common import initializer as init def _calculate_gain(nonlinearity, param=None): @@ -191,23 +190,25 @@ def default_recurisive_init(custom_cell): for _, cell in custom_cell.cells_and_names(): if isinstance(cell, nn.Conv2d): cell.weight.default_input = init.initializer(KaimingUniform(a=math.sqrt(5)), - cell.weight.default_input.shape, - cell.weight.default_input.dtype).to_tensor() + cell.weight.shape, + cell.weight.dtype) if cell.bias is not None: - fan_in, _ = _calculate_in_and_out(cell.weight.default_input.asnumpy()) + fan_in, _ = _calculate_in_and_out(cell.weight) bound = 1 / math.sqrt(fan_in) np.random.seed(0) - cell.bias.default_input = Tensor(np.random.uniform(-bound, bound, cell.bias.default_input.shape), - cell.bias.default_input.dtype) + cell.bias.default_input = init.initializer(init.Uniform(bound), + cell.bias.shape, + cell.bias.dtype) elif isinstance(cell, nn.Dense): cell.weight.default_input = init.initializer(KaimingUniform(a=math.sqrt(5)), - cell.weight.default_input.shape, - cell.weight.default_input.dtype).to_tensor() + cell.weight.shape, + cell.weight.dtype) if cell.bias is not None: - fan_in, _ = _calculate_in_and_out(cell.weight.default_input.asnumpy()) + fan_in, _ = _calculate_in_and_out(cell.weight) bound = 1 / math.sqrt(fan_in) np.random.seed(0) - cell.bias.default_input = Tensor(np.random.uniform(-bound, bound, cell.bias.default_input.shape), - cell.bias.default_input.dtype) + cell.bias.default_input = init.initializer(init.Uniform(bound), + cell.bias.shape, + cell.bias.dtype) elif isinstance(cell, (nn.BatchNorm2d, nn.BatchNorm1d)): pass diff --git a/model_zoo/official/cv/vgg16/src/vgg.py b/model_zoo/official/cv/vgg16/src/vgg.py index dc8c9e698d..bd873e4d5c 100644 --- a/model_zoo/official/cv/vgg16/src/vgg.py +++ b/model_zoo/official/cv/vgg16/src/vgg.py @@ -102,16 +102,16 @@ class Vgg(nn.Cell): if isinstance(cell, nn.Conv2d): cell.weight.default_input = init.initializer( KaimingNormal(a=math.sqrt(5), mode='fan_out', nonlinearity='relu'), - cell.weight.default_input.shape, cell.weight.default_input.dtype).to_tensor() + cell.weight.shape, cell.weight.dtype) if cell.bias is not None: cell.bias.default_input = init.initializer( - 'zeros', cell.bias.default_input.shape, cell.bias.default_input.dtype).to_tensor() + 'zeros', cell.bias.shape, cell.bias.dtype) elif isinstance(cell, nn.Dense): cell.weight.default_input = init.initializer( - init.Normal(0.01), cell.weight.default_input.shape, cell.weight.default_input.dtype).to_tensor() + init.Normal(0.01), cell.weight.shape, cell.weight.dtype) if cell.bias is not None: cell.bias.default_input = init.initializer( - 'zeros', cell.bias.default_input.shape, cell.bias.default_input.dtype).to_tensor() + 'zeros', cell.bias.shape, cell.bias.dtype) cfg = { diff --git a/model_zoo/official/cv/yolov3_darknet53/src/initializer.py b/model_zoo/official/cv/yolov3_darknet53/src/initializer.py index f3c03a8ad1..c66cc74acf 100644 --- a/model_zoo/official/cv/yolov3_darknet53/src/initializer.py +++ b/model_zoo/official/cv/yolov3_darknet53/src/initializer.py @@ -14,11 +14,11 @@ # ============================================================================ """Parameter init.""" import math +from functools import reduce import numpy as np from mindspore.common import initializer as init from mindspore.common.initializer import Initializer as MeInitializer import mindspore.nn as nn -from mindspore import Tensor np.random.seed(5) @@ -134,7 +134,7 @@ def _calculate_fan_in_and_fan_out(arr): num_output_fmaps = arr.shape[0] receptive_field_size = 1 if dimensions > 2: - receptive_field_size = arr[0][0].size + receptive_field_size = reduce(lambda x, y: x * y, arr.shape[2:]) fan_in = num_input_fmaps * receptive_field_size fan_out = num_output_fmaps * receptive_field_size @@ -159,21 +159,23 @@ def default_recurisive_init(custom_cell): for _, cell in custom_cell.cells_and_names(): if isinstance(cell, nn.Conv2d): cell.weight.default_input = init.initializer(KaimingUniform(a=math.sqrt(5)), - cell.weight.default_input.shape, - cell.weight.default_input.dtype).to_tensor() + cell.weight.shape, + cell.weight.dtype) if cell.bias is not None: - fan_in, _ = _calculate_fan_in_and_fan_out(cell.weight.default_input.asnumpy()) + fan_in, _ = _calculate_fan_in_and_fan_out(cell.weight) bound = 1 / math.sqrt(fan_in) - cell.bias.default_input = Tensor(np.random.uniform(-bound, bound, cell.bias.default_input.shape), - cell.bias.default_input.dtype) + cell.bias.default_input = init.initializer(init.Uniform(bound), + cell.bias.shape, + cell.bias.dtype) elif isinstance(cell, nn.Dense): cell.weight.default_input = init.initializer(KaimingUniform(a=math.sqrt(5)), - cell.weight.default_input.shape, - cell.weight.default_input.dtype).to_tensor() + cell.weight.shape, + cell.weight.dtype) if cell.bias is not None: - fan_in, _ = _calculate_fan_in_and_fan_out(cell.weight.default_input.asnumpy()) + fan_in, _ = _calculate_fan_in_and_fan_out(cell.weight) bound = 1 / math.sqrt(fan_in) - cell.bias.default_input = Tensor(np.random.uniform(-bound, bound, cell.bias.default_input.shape), - cell.bias.default_input.dtype) + cell.bias.default_input = init.initializer(init.Uniform(bound), + cell.bias.shape, + cell.bias.dtype) elif isinstance(cell, (nn.BatchNorm2d, nn.BatchNorm1d)): pass diff --git a/model_zoo/official/cv/yolov3_darknet53/src/util.py b/model_zoo/official/cv/yolov3_darknet53/src/util.py index 1a3da99181..f97bdd2548 100644 --- a/model_zoo/official/cv/yolov3_darknet53/src/util.py +++ b/model_zoo/official/cv/yolov3_darknet53/src/util.py @@ -58,7 +58,7 @@ def load_backbone(net, ckpt_path, args): darknet_backbone_prefix = 'network.backbone' find_param = [] not_found_param = [] - + net.init_parameters_data() for name, cell in net.cells_and_names(): if name.startswith(yolo_backbone_prefix): name = name.replace(yolo_backbone_prefix, darknet_backbone_prefix)