m53297601
/
mindspore
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fix pylint warnings in mindspore st test module

Browse Source
pull/577/head
leonwanghui 5 years ago
parent ca3aa6071a
commit ba43dbc148
38 changed files with 317 additions and 252 deletions
Show Stats Download Patch File Download Diff File

38
tests/st/auto_parallel/onehot_model_parallel.py
Unescape View File

@ -11,6 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
# ============================================================================
import os import os
import pytest import pytest
@ -26,6 +27,7 @@ device_num = 2
device_id = int(os.getenv('DEVICE_ID')) device_id = int(os.getenv('DEVICE_ID'))
rank_id = 0 rank_id = 0
def setup_module(): def setup_module():
global device_num global device_num
global rank_id global rank_id
@ -42,9 +44,11 @@ def setup_module():
context.set_auto_parallel_context(device_num=device_num, context.set_auto_parallel_context(device_num=device_num,
global_rank=rank_id) global_rank=rank_id)
def teardown_module(): def teardown_module():
distributedTool.release() distributedTool.release()
class Onehot(Cell): class Onehot(Cell):
def __init__(self, axis=-1, depth=1, on_value=1.0, off_value=0.0, strategy=None): def __init__(self, axis=-1, depth=1, on_value=1.0, off_value=0.0, strategy=None):
super(Onehot, self).__init__() super(Onehot, self).__init__()
@ -56,25 +60,26 @@ class Onehot(Cell):
self.on_value = Tensor(on_value, ms.float32) self.on_value = Tensor(on_value, ms.float32)
self.off_value = Tensor(off_value, ms.float32) self.off_value = Tensor(off_value, ms.float32)
self.transpose = P.Transpose().set_strategy(strategy=trans_stra) self.transpose = P.Transpose().set_strategy(strategy=trans_stra)
self.sub = P.Sub().set_strategy(strategy=((1,1),(1,1))) self.sub = P.Sub().set_strategy(strategy=((1, 1), (1, 1)))
def construct(self, input, indices): def construct(self, input, indices):
x = self.onehot(indices, self.depth, self.on_value, self.off_value) x = self.onehot(indices, self.depth, self.on_value, self.off_value)
x = self.transpose(x, (1,0)) x = self.transpose(x, (1, 0))
x = self.sub(input, x) x = self.sub(input, x)
return x return x
class DataGenerator(): class DataGenerator():
def get_parallel_blocks(self, input_, strategy): def get_parallel_blocks(self, input_, strategy):
blocks = [input_] blocks = [input_]
i = 0 i = 0
for stra in strategy: for stra in strategy:
temp = [] temp = []
while len(blocks)>0: while len(blocks) > 0:
block = blocks.pop(0) block = blocks.pop(0)
temp.extend(np.split(block, stra, axis=i)) temp.extend(np.split(block, stra, axis=i))
blocks.extend(temp) blocks.extend(temp)
i+=1 i += 1
return blocks return blocks
def generate_data(self, shape): def generate_data(self, shape):
@ -93,32 +98,33 @@ class DataGenerator():
stra = [1]*len(shape) stra = [1]*len(shape)
stra[0] = device_num stra[0] = device_num
datas = self.get_parallel_blocks(data, stra) datas = self.get_parallel_blocks(data, stra)
return Tensor(data),Tensor(datas[rank_id]) return Tensor(data), Tensor(datas[rank_id])
class OneHotFactory: class OneHotFactory:
def __init__(self, batch_size, classes, on_value=1.0, off_value=0.0, axis=None, strategy=None): def __init__(self, batch_size, classes, on_value=1.0, off_value=0.0, axis=None, strategy=None):
dataGen = DataGenerator() dataGen = DataGenerator()
self.input_full, self.input_part = dataGen.input_data((classes, batch_size)) self.input_full, self.input_part = dataGen.input_data((classes, batch_size))
self.label_full, self.label_part = dataGen.label_data((batch_size,),classes) self.label_full, self.label_part = dataGen.label_data((batch_size,), classes)
self.depth = classes self.depth = classes
self.on_value = on_value self.on_value = on_value
self.off_value = off_value self.off_value = off_value
self.axis = axis self.axis = axis
self.strategy = strategy self.strategy = strategy
def forward_mindspore_single_impl(self): def forward_mindspore_single_impl(self):
net = Onehot(axis=self.axis, net = Onehot(axis=self.axis,
depth=self.depth, depth=self.depth,
on_value=self.on_value, on_value=self.on_value,
off_value=self.off_value) off_value=self.off_value)
out = net(self.input_full, self.label_full) out = net(self.input_full, self.label_full)
return out return out
def forward_mindspore_parallel_impl(self): def forward_mindspore_parallel_impl(self):
context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") context.set_auto_parallel_context(parallel_mode="semi_auto_parallel")
net = Onehot(axis=self.axis, net = Onehot(axis=self.axis,
depth=self.depth, depth=self.depth,
on_value=self.on_value, on_value=self.on_value,
off_value=self.off_value, strategy=self.strategy) off_value=self.off_value, strategy=self.strategy)
out = net.compile_and_run(self.input_full, self.label_full) out = net.compile_and_run(self.input_full, self.label_full)
return out return out
@ -137,7 +143,7 @@ def test_reid_onehot_forward_int32_128_depth1024_model_parallel():
on_value=1.000000, on_value=1.000000,
off_value=0.000000, off_value=0.000000,
axis=-1, axis=-1,
strategy=((1,device_num),(),())) strategy=((1, device_num), (), ()))
fact.forward_cmp() fact.forward_cmp()
@ -147,5 +153,5 @@ def test_reid_onehot_forward_int32_1024_depth128_model_parallel():
on_value=1.000000, on_value=1.000000,
off_value=0.000000, off_value=0.000000,
axis=-1, axis=-1,
strategy=((1,device_num),(),())) strategy=((1, device_num), (), ()))
fact.forward_cmp() fact.forward_cmp()

90
tests/st/auto_parallel/soft_entropy_loss_expand_parallel.py
Unescape View File

@ -11,6 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
# ============================================================================
import os import os
import pytest import pytest
@ -31,7 +32,7 @@ from mindspore.nn.optim.momentum import Momentum
from mindspore.train.callback import Callback from mindspore.train.callback import Callback
np.set_printoptions(threshold=np.inf) np.set_printoptions(threshold=np.inf)
device_num=2 device_num = 2
device_id = int(os.getenv('DEVICE_ID')) device_id = int(os.getenv('DEVICE_ID'))
rank_id = 0 rank_id = 0
embed = 128 embed = 128
@ -39,6 +40,7 @@ classes = 32
batch_size = 32*2 batch_size = 32*2
MatmulParamShape = (classes, embed) MatmulParamShape = (classes, embed)
def setup_module(): def setup_module():
global device_num global device_num
global rank_id global rank_id
@ -55,26 +57,28 @@ def setup_module():
context.set_auto_parallel_context(device_num=device_num, context.set_auto_parallel_context(device_num=device_num,
global_rank=device_id) global_rank=device_id)
def teardown_module(): def teardown_module():
distributedTool.release() distributedTool.release()
class DataGenerator(): class DataGenerator():
def get_parallel_blocks(self, input_, strategy): def get_parallel_blocks(self, input_, strategy):
blocks = [input_] blocks = [input_]
i = 0 i = 0
for stra in strategy: for stra in strategy:
temp = [] temp = []
while len(blocks)>0: while len(blocks) > 0:
block = blocks.pop(0) block = blocks.pop(0)
temp.extend(np.split(block, stra, axis=i)) temp.extend(np.split(block, stra, axis=i))
blocks.extend(temp) blocks.extend(temp)
i+=1 i += 1
return blocks return blocks
def generate_data(self, shape): def generate_data(self, shape):
size = np.cumprod(shape)[-1] size = np.cumprod(shape)[-1]
num_range = min(size, 1000) num_range = min(size, 1000)
data = (np.arange(0, size)%num_range)/num_range data = (np.arange(0, size) % num_range)/num_range
data = np.reshape(data, shape) data = np.reshape(data, shape)
return data return data
@ -83,14 +87,15 @@ class DataGenerator():
stra = [1]*len(shape) stra = [1]*len(shape)
stra[0] = device_num stra[0] = device_num
datas = self.get_parallel_blocks(data, stra) datas = self.get_parallel_blocks(data, stra)
return Tensor(data), Tensor(datas[rank_id]) return Tensor(data), Tensor(datas[rank_id])
def label_data(self, shape, embed): def label_data(self, shape, embed):
data = (self.generate_data(shape)*(embed-1)).astype(np.int32) data = (self.generate_data(shape)*(embed-1)).astype(np.int32)
stra = [1]*len(shape) stra = [1]*len(shape)
stra[0] = device_num stra[0] = device_num
datas = self.get_parallel_blocks(data, stra) datas = self.get_parallel_blocks(data, stra)
return Tensor(data),Tensor(datas[rank_id]) return Tensor(data), Tensor(datas[rank_id])
class Dataset(): class Dataset():
def __init__(self, predict, label, length=1, input_num=2): def __init__(self, predict, label, length=1, input_num=2):
@ -121,15 +126,18 @@ class Dataset():
def get_repeat_count(self): def get_repeat_count(self):
return self.length return self.length
class ModelCallback(Callback): class ModelCallback(Callback):
def __init__(self): def __init__(self):
super(ModelCallback, self).__init__() super(ModelCallback, self).__init__()
self.loss_list = [] self.loss_list = []
def epoch_end(self, run_context, *args): def epoch_end(self, run_context, *args):
cb_params = run_context.original_args() cb_params = run_context.original_args()
result = cb_params.net_outputs result = cb_params.net_outputs
self.loss_list.append(result.asnumpy().mean()) self.loss_list.append(result.asnumpy().mean())
class SoftmaxCrossEntropyExpand(Cell): class SoftmaxCrossEntropyExpand(Cell):
def __init__(self, sparse=False, stra_list=[]): def __init__(self, sparse=False, stra_list=[]):
super(SoftmaxCrossEntropyExpand, self).__init__() super(SoftmaxCrossEntropyExpand, self).__init__()
@ -164,22 +172,25 @@ class SoftmaxCrossEntropyExpand(Cell):
loss = self.reduce_mean(loss, -1) loss = self.reduce_mean(loss, -1)
return loss return loss
class MatmulNet(Cell): class MatmulNet(Cell):
def __init__(self, matmul_stra = None, loss_stra_list=[]): def __init__(self, matmul_stra=None, loss_stra_list=[]):
super(MatmulNet, self).__init__() super(MatmulNet, self).__init__()
self.matmul = P.MatMul(transpose_b=True).set_strategy(strategy=matmul_stra) self.matmul = P.MatMul(transpose_b=True).set_strategy(strategy=matmul_stra)
self.loss = SoftmaxCrossEntropyExpand(sparse=True, stra_list=loss_stra_list) self.loss = SoftmaxCrossEntropyExpand(sparse=True, stra_list=loss_stra_list)
self.weight = Parameter(Tensor(np.ones(MatmulParamShape), dtype=ms.float32), name="weight") self.weight = Parameter(Tensor(np.ones(MatmulParamShape), dtype=ms.float32), name="weight")
def construct(self, x, label): def construct(self, x, label):
loss_input = self.matmul(x, self.weight) loss_input = self.matmul(x, self.weight)
out = self.loss(loss_input, label) out = self.loss(loss_input, label)
return out return out
class LossFactory(): class LossFactory():
def __init__(self): def __init__(self):
dataGen = DataGenerator() dataGen = DataGenerator()
self.input_full, self.input_part = dataGen.input_data((batch_size, embed)) self.input_full, self.input_part = dataGen.input_data((batch_size, embed))
self.label_full, self.label_part = dataGen.label_data((batch_size,),embed) self.label_full, self.label_part = dataGen.label_data((batch_size,), embed)
def single_matmul_trains(self): def single_matmul_trains(self):
single_callback = ModelCallback() single_callback = ModelCallback()
@ -196,32 +207,33 @@ class LossFactory():
parallel_callback = ModelCallback() parallel_callback = ModelCallback()
context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") context.set_auto_parallel_context(parallel_mode="semi_auto_parallel")
net = MatmulNet() net = MatmulNet()
optimizer = Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9) optimizer = Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9)
model = Model(net, optimizer=optimizer) model = Model(net, optimizer=optimizer)
epoch_size = 6 epoch_size = 6
dataset = Dataset(self.input_part, self.label_part) dataset = Dataset(self.input_part, self.label_part)
model.train(epoch_size, dataset, callbacks=parallel_callback, dataset_sink_mode=False) model.train(epoch_size, dataset, callbacks=parallel_callback, dataset_sink_mode=False)
loss_value = np.array(parallel_callback.loss_list) loss_value = np.array(parallel_callback.loss_list)
return loss_value return loss_value
def model_parallel_matmul_trains(self): def model_parallel_matmul_trains(self):
parallel_callback = ModelCallback() parallel_callback = ModelCallback()
matmul_stra = ((1,1),(device_num,1)) matmul_stra = ((1, 1), (device_num, 1))
reduce_max_stra = ((1,device_num),) reduce_max_stra = ((1, device_num),)
sub_stra = ((1,device_num),(1,1)) sub_stra = ((1, device_num), (1, 1))
exp_stra = ((1,device_num),) exp_stra = ((1, device_num),)
reduce_sum_stra = ((1,device_num),) reduce_sum_stra = ((1, device_num),)
div_stra = ((1,device_num),(1,1)) div_stra = ((1, device_num), (1, 1))
log_stra = ((1,device_num),) log_stra = ((1, device_num),)
mul_stra = ((1,device_num),(1,device_num)) mul_stra = ((1, device_num), (1, device_num))
sum_cross_entropy_stra = ((1,device_num),) sum_cross_entropy_stra = ((1, device_num),)
mul2_stra = ((),(device_num,)) mul2_stra = ((), (device_num,))
reduce_mean_stra = ((device_num,),) reduce_mean_stra = ((device_num,),)
onehot_stra = ((1,device_num),(),()) onehot_stra = ((1, device_num), (), ())
loss_stra_list = [exp_stra, reduce_sum_stra, onehot_stra, div_stra, log_stra, sum_cross_entropy_stra, mul_stra, mul2_stra, reduce_mean_stra, reduce_max_stra, sub_stra] loss_stra_list = [exp_stra, reduce_sum_stra, onehot_stra, div_stra, log_stra,
sum_cross_entropy_stra, mul_stra, mul2_stra, reduce_mean_stra, reduce_max_stra, sub_stra]
context.set_auto_parallel_context(parallel_mode="auto_parallel") context.set_auto_parallel_context(parallel_mode="auto_parallel")
net = MatmulNet(matmul_stra = matmul_stra, loss_stra_list = loss_stra_list) net = MatmulNet(matmul_stra=matmul_stra, loss_stra_list=loss_stra_list)
optimizer = Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9) optimizer = Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9)
model = Model(net, optimizer=optimizer) model = Model(net, optimizer=optimizer)
epoch_size = 6 epoch_size = 6
dataset = Dataset(self.input_part, self.label_part) dataset = Dataset(self.input_part, self.label_part)
@ -231,21 +243,22 @@ class LossFactory():
def mix_parallel_matmul_trains(self): def mix_parallel_matmul_trains(self):
parallel_callback = ModelCallback() parallel_callback = ModelCallback()
matmul_stra = ((device_num,1),(1,1)) matmul_stra = ((device_num, 1), (1, 1))
reduce_max_stra = ((1,device_num),) reduce_max_stra = ((1, device_num),)
sub_stra = ((device_num,1),(device_num,1)) sub_stra = ((device_num, 1), (device_num, 1))
exp_stra = ((1,device_num),) exp_stra = ((1, device_num),)
reduce_sum_stra = ((1,device_num),) reduce_sum_stra = ((1, device_num),)
div_stra = ((1,device_num),(1,1)) div_stra = ((1, device_num), (1, 1))
log_stra = ((1,device_num),) log_stra = ((1, device_num),)
mul_stra = ((1,device_num),(1,device_num)) mul_stra = ((1, device_num), (1, device_num))
sum_cross_entropy_stra = ((1,device_num),) sum_cross_entropy_stra = ((1, device_num),)
mul2_stra = ((),(device_num,)) mul2_stra = ((), (device_num,))
reduce_mean_stra = ((device_num,),) reduce_mean_stra = ((device_num,),)
onehot_stra = ((1,device_num),(),()) onehot_stra = ((1, device_num), (), ())
loss_stra_list = [exp_stra, reduce_sum_stra, onehot_stra, div_stra, log_stra, sum_cross_entropy_stra, mul_stra, mul2_stra, reduce_mean_stra, reduce_max_stra, sub_stra] loss_stra_list = [exp_stra, reduce_sum_stra, onehot_stra, div_stra, log_stra,
sum_cross_entropy_stra, mul_stra, mul2_stra, reduce_mean_stra, reduce_max_stra, sub_stra]
context.set_auto_parallel_context(parallel_mode="auto_parallel") context.set_auto_parallel_context(parallel_mode="auto_parallel")
net = MatmulNet(matmul_stra = matmul_stra, loss_stra_list = loss_stra_list) net = MatmulNet(matmul_stra=matmul_stra, loss_stra_list=loss_stra_list)
optimizer = Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9) optimizer = Momentum(net.trainable_params(), learning_rate=0.1, momentum=0.9)
model = Model(net, optimizer=optimizer) model = Model(net, optimizer=optimizer)
epoch_size = 6 epoch_size = 6
@ -254,6 +267,7 @@ class LossFactory():
loss_value = np.array(parallel_callback.loss_list) loss_value = np.array(parallel_callback.loss_list)
return loss_value return loss_value
def test_all_trains(): def test_all_trains():
loss_factory = LossFactory() loss_factory = LossFactory()
context.reset_auto_parallel_context() context.reset_auto_parallel_context()

4
tests/st/auto_parallel/test_expand_loss.py
Unescape View File

@ -12,10 +12,10 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
# ============================================================================ # ============================================================================
import os import os
import pytest import pytest
@pytest.mark.level0 @pytest.mark.level0
@pytest.mark.platform_x86_ascend_training @pytest.mark.platform_x86_ascend_training
@pytest.mark.platform_arm_ascend_training @pytest.mark.platform_arm_ascend_training
@ -23,4 +23,4 @@ import pytest
def test_expand_loss(): def test_expand_loss():
sh_path = os.path.split(os.path.realpath(__file__))[0] sh_path = os.path.split(os.path.realpath(__file__))[0]
ret = os.system(f"sh {sh_path}/run_auto_parallel_loss_expand.sh") ret = os.system(f"sh {sh_path}/run_auto_parallel_loss_expand.sh")
assert(ret==0) assert(ret == 0)

3
tests/st/auto_parallel/test_model_parallel_onehot.py
Unescape View File

@ -16,6 +16,7 @@
import os import os
import pytest import pytest
def test_expand_loss(): def test_expand_loss():
ret = os.system("sh run_onehot_model_parallel.sh") ret = os.system("sh run_onehot_model_parallel.sh")
assert(ret==0) assert(ret == 0)

27
tests/st/auto_parallel/test_resnet50_expand_loss_2p.py
Unescape View File

@ -11,10 +11,12 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
# ============================================================================
import os
import numpy as np import numpy as np
import pytest import pytest
from numpy import allclose import mindspore.context as context
import mindspore.nn as nn import mindspore.nn as nn
import mindspore.common.dtype as mstype import mindspore.common.dtype as mstype
from mindspore import Tensor from mindspore import Tensor
@ -22,21 +24,21 @@ from mindspore.ops import operations as P
from mindspore.nn.optim.momentum import Momentum from mindspore.nn.optim.momentum import Momentum
from mindspore.common.initializer import One from mindspore.common.initializer import One
from mindspore.train.model import Model, ParallelMode from mindspore.train.model import Model, ParallelMode
from mindspore import context
import os
from mindspore.communication.management import init from mindspore.communication.management import init
import mindspore.ops.functional as F import mindspore.ops.functional as F
from mindspore.nn.loss.loss import _Loss from mindspore.nn.loss.loss import _Loss
from mindspore.train.callback import Callback from mindspore.train.callback import Callback
from mindspore.parallel import set_algo_parameters from mindspore.parallel import set_algo_parameters
context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") context.set_context(mode=context.GRAPH_MODE, device_target="Ascend")
context.set_context(enable_hccl=True) context.set_context(enable_hccl=True)
context.set_context(enable_task_sink=True,device_id=int(os.getenv('DEVICE_ID'))) context.set_context(enable_task_sink=True, device_id=int(os.getenv('DEVICE_ID')))
context.set_context(enable_ir_fusion=True) context.set_context(enable_ir_fusion=True)
context.set_context(enable_loop_sink=False) context.set_context(enable_loop_sink=False)
init() init()
context.set_auto_parallel_context(mirror_mean=True, parallel_mode=ParallelMode.AUTO_PARALLEL) context.set_auto_parallel_context(mirror_mean=True, parallel_mode=ParallelMode.AUTO_PARALLEL)
def weight_variable(shape, factor=0.1): def weight_variable(shape, factor=0.1):
return One() return One()
@ -52,6 +54,7 @@ def _conv1x1(in_channels, out_channels, stride=1, padding=0, pad_mode='same'):
return nn.Conv2d(in_channels, out_channels, return nn.Conv2d(in_channels, out_channels,
kernel_size=1, stride=stride, padding=padding, pad_mode=pad_mode, weight_init=init_value) kernel_size=1, stride=stride, padding=padding, pad_mode=pad_mode, weight_init=init_value)
def _conv7x7(in_channels, out_channels, stride=1, padding=0, pad_mode='same'): def _conv7x7(in_channels, out_channels, stride=1, padding=0, pad_mode='same'):
init_value = weight_variable((out_channels, in_channels, 7, 7)) init_value = weight_variable((out_channels, in_channels, 7, 7))
return nn.Conv2d(in_channels, out_channels, return nn.Conv2d(in_channels, out_channels,
@ -63,6 +66,7 @@ def _fused_bn(channels, momentum=0.9):
init_bias = weight_variable((channels,)) init_bias = weight_variable((channels,))
return nn.BatchNorm2d(channels, momentum=momentum) return nn.BatchNorm2d(channels, momentum=momentum)
class BasicBlock(nn.Cell): class BasicBlock(nn.Cell):
expansion = 1 expansion = 1
@ -172,7 +176,7 @@ class ResNet(nn.Cell):
layer_nums, layer_nums,
in_channels, in_channels,
out_channels, out_channels,
strides=[1,2,2,2], strides=[1, 2, 2, 2],
num_classes=100): num_classes=100):
super(ResNet, self).__init__() super(ResNet, self).__init__()
@ -292,17 +296,19 @@ class SoftmaxCrossEntropyExpand(_Loss):
rank_id = int(os.environ["RANK_ID"]) rank_id = int(os.environ["RANK_ID"])
device_num = int(os.environ["RANK_SIZE"]) device_num = int(os.environ["RANK_SIZE"])
class DataGenerator(): class DataGenerator():
def get_parallel_blocks(self, input_, strategy): def get_parallel_blocks(self, input_, strategy):
blocks = [input_] blocks = [input_]
i = 0 i = 0
for stra in strategy: for stra in strategy:
temp = [] temp = []
while len(blocks)>0: while len(blocks) > 0:
block = blocks.pop(0) block = blocks.pop(0)
temp.extend(np.split(block, stra, axis=i)) temp.extend(np.split(block, stra, axis=i))
blocks.extend(temp) blocks.extend(temp)
i+=1 i += 1
return blocks return blocks
def generate_data(self, shape): def generate_data(self, shape):
@ -321,7 +327,7 @@ class DataGenerator():
stra = [1]*len(shape) stra = [1]*len(shape)
stra[0] = device_num stra[0] = device_num
datas = self.get_parallel_blocks(data, stra) datas = self.get_parallel_blocks(data, stra)
return Tensor(data),Tensor(datas[rank_id]) return Tensor(data), Tensor(datas[rank_id])
class Dataset(): class Dataset():
@ -359,6 +365,7 @@ class ModelCallback(Callback):
def __init__(self): def __init__(self):
super(ModelCallback, self).__init__() super(ModelCallback, self).__init__()
self.loss_list = [] self.loss_list = []
def epoch_end(self, run_context, *args): def epoch_end(self, run_context, *args):
cb_params = run_context.original_args() cb_params = run_context.original_args()
result = cb_params.net_outputs result = cb_params.net_outputs
@ -382,7 +389,7 @@ def test_train_feed(num_classes=8192):
model.train(5, dataset, dataset_sink_mode=False, callbacks=parallel_callback) model.train(5, dataset, dataset_sink_mode=False, callbacks=parallel_callback)
loss_value = np.array(parallel_callback.loss_list) loss_value = np.array(parallel_callback.loss_list)
expect_out = [9.010913, 8.855984, 8.56246, 8.146317, 7.624489] expect_out = [9.010913, 8.855984, 8.56246, 8.146317, 7.624489]
assert allclose(loss_value, expect_out, 0.0001, 0.0001) assert np.allclose(loss_value, expect_out, 0.0001, 0.0001)
@pytest.mark.level0 @pytest.mark.level0
@ -402,4 +409,4 @@ def test_train_feed2(num_classes=1001):
model.train(5, dataset, dataset_sink_mode=False, callbacks=parallel_callback) model.train(5, dataset, dataset_sink_mode=False, callbacks=parallel_callback)
loss_value = np.array(parallel_callback.loss_list) loss_value = np.array(parallel_callback.loss_list)
expect_out = [6.908755, 6.8358116, 6.6986914, 6.506859, 6.2708097] expect_out = [6.908755, 6.8358116, 6.6986914, 6.506859, 6.2708097]
assert allclose(loss_value, expect_out, 0.0001, 0.0001) assert np.allclose(loss_value, expect_out, 0.0001, 0.0001)

14
tests/st/control/test_while.py
Unescape View File

@ -13,12 +13,12 @@
# limitations under the License. # limitations under the License.
# ============================================================================ # ============================================================================
import numpy as np import numpy as np
from mindspore.common.tensor import Tensor
from mindspore.common import dtype as mstype
import mindspore.context as context import mindspore.context as context
from mindspore.ops import operations as P
import mindspore.nn as nn import mindspore.nn as nn
from mindspore.common import ms_function from mindspore import Tensor, ms_function
from mindspore.common import dtype as mstype
from mindspore.ops import operations as P
@ms_function @ms_function
def t1_while(x, y, z): def t1_while(x, y, z):
@ -28,8 +28,9 @@ def t1_while(x, y, z):
x = x + 3 x = x + 3
return x return x
def test_net(): def test_net():
context.set_context(mode=context.GRAPH_MODE,device_target="Ascend") context.set_context(mode=context.GRAPH_MODE, device_target="Ascend")
context.set_context(enable_task_sink=True) context.set_context(enable_task_sink=True)
c1 = Tensor([2], mstype.int32) c1 = Tensor([2], mstype.int32)
c2 = Tensor([14], mstype.int32) c2 = Tensor([14], mstype.int32)
@ -38,5 +39,6 @@ def test_net():
ret = t1_while(c1, c2, c3) ret = t1_while(c1, c2, c3)
assert (ret == expect) assert (ret == expect)
if __name__ == "__main__": if __name__ == "__main__":
test_net() test_net()

20
tests/st/fusion/test_add_relu_buffer_fusion.py
Unescape View File

@ -12,17 +12,16 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
# ============================================================================ # ============================================================================
from mindspore import Tensor
from mindspore.ops import operations as P
import mindspore.nn as nn
from mindspore.common.api import ms_function
import mindspore.common.dtype as mstype
import numpy as np import numpy as np
import mindspore.context as context import mindspore.context as context
from mindspore.common.initializer import initializer import mindspore.nn as nn
from mindspore.common.parameter import Parameter import mindspore.common.dtype as mstype
from mindspore import Tensor, ms_function
from mindspore.ops import operations as P
context.set_context(mode=context.GRAPH_MODE, device_id=5, device_target="Ascend") context.set_context(mode=context.GRAPH_MODE, device_id=5, device_target="Ascend")
#context.set_context(enable_task_sink=True)
class Net(nn.Cell): class Net(nn.Cell):
def __init__(self): def __init__(self):
super(Net, self).__init__() super(Net, self).__init__()
@ -35,17 +34,14 @@ class Net(nn.Cell):
def construct(self, x, y): def construct(self, x, y):
x = self.cast(x, mstype.float16) x = self.cast(x, mstype.float16)
y = self.cast(y, mstype.float16) y = self.cast(y, mstype.float16)
#x = self.softmax(x)
x = self.add(x, y) x = self.add(x, y)
#x = self.relu(x)
x = self.relu(x) x = self.relu(x)
#x = self.softmax(x)
x = self.reduce_mean(x) x = self.reduce_mean(x)
return x return x
def test_net(): def test_net():
x = np.random.randn(32, 10).astype(np.float32) x = np.random.randn(32, 10).astype(np.float32)
relu = Net() relu = Net()
output = relu(Tensor(x), Tensor(x)) output = relu(Tensor(x), Tensor(x))
print(x)
print(output.asnumpy()) print(output.asnumpy())

25
tests/st/fusion/test_conv_bn1_fusion.py
Unescape View File

@ -13,15 +13,13 @@
# limitations under the License. # limitations under the License.
# ============================================================================ # ============================================================================
import numpy as np import numpy as np
import mindspore.context as context
import mindspore.nn as nn import mindspore.nn as nn
from mindspore import Tensor, Parameter, Model, ms_function
from mindspore.ops import operations as P from mindspore.ops import operations as P
from mindspore.common.initializer import initializer from mindspore.common.initializer import initializer
from mindspore import Tensor, Parameter, Model
from mindspore.nn.loss import SoftmaxCrossEntropyWithLogits from mindspore.nn.loss import SoftmaxCrossEntropyWithLogits
from mindspore.nn.optim import Momentum from mindspore.nn.optim import Momentum
from mindspore.common.api import ms_function
import mindspore.nn as wrap
import mindspore.context as context
context.set_context(device_target="Ascend", enable_task_sink=True) context.set_context(device_target="Ascend", enable_task_sink=True)
@ -35,6 +33,7 @@ class MsWrapper(nn.Cell):
def __init__(self, network): def __init__(self, network):
super(MsWrapper, self).__init__(auto_prefix=False) super(MsWrapper, self).__init__(auto_prefix=False)
self._network = network self._network = network
@ms_function @ms_function
def construct(self, *args): def construct(self, *args):
return self._network(*args) return self._network(*args)
@ -42,16 +41,16 @@ class MsWrapper(nn.Cell):
def me_train_tensor(net, input_np, label_np, epoch_size=2): def me_train_tensor(net, input_np, label_np, epoch_size=2):
loss = SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True) loss = SoftmaxCrossEntropyWithLogits(is_grad=False, sparse=True)
opt = nn.Momentum(Tensor(np.array([0.1])), Tensor(np.array([0.9])), filter(lambda x: x.requires_grad, net.get_parameters())) opt = nn.Momentum(Tensor(np.array([0.1])), Tensor(np.array([0.9])),
filter(lambda x: x.requires_grad, net.get_parameters()))
context.set_context(mode=context.GRAPH_MODE) context.set_context(mode=context.GRAPH_MODE)
Model(net, loss, opt) Model(net, loss, opt)
_network = wrap.WithLossCell(net, loss) _network = nn.WithLossCell(net, loss)
_train_net = MsWrapper(wrap.TrainOneStepCell(_network, opt)) _train_net = MsWrapper(nn.TrainOneStepCell(_network, opt))
_train_net.set_train() _train_net.set_train()
for epoch in range(0, epoch_size): for epoch in range(0, epoch_size):
print(f"epoch %d"%(epoch)) print(f"epoch %d" % (epoch))
output = _train_net(Tensor(input_np), Tensor(label_np)) output = _train_net(Tensor(input_np), Tensor(label_np))
print("********output***********")
print(output.asnumpy()) print(output.asnumpy())
@ -60,9 +59,9 @@ def test_conv_bn_add_relu_fusion():
def __init__(self): def __init__(self):
super(Net, self).__init__() super(Net, self).__init__()
self.conv = nn.Conv2d(input_channel, output_channel, self.conv = nn.Conv2d(input_channel, output_channel,
kernel_size=1, stride=1, padding=0, has_bias=False, pad_mode="same") kernel_size=1, stride=1, padding=0, has_bias=False, pad_mode="same")
self.conv1 = nn.Conv2d(input_channel, output_channel, self.conv1 = nn.Conv2d(input_channel, output_channel,
kernel_size=1, stride=1, padding=0, has_bias=False, pad_mode="same") kernel_size=1, stride=1, padding=0, has_bias=False, pad_mode="same")
self.bn = nn.BatchNorm2d(output_channel, momentum=0.1, eps=0.0001) self.bn = nn.BatchNorm2d(output_channel, momentum=0.1, eps=0.0001)
self.add = P.TensorAdd() self.add = P.TensorAdd()
self.relu = P.ReLU() self.relu = P.ReLU()
@ -91,7 +90,7 @@ def test_conv_bn_relu_fusion():
def __init__(self): def __init__(self):
super(Net, self).__init__() super(Net, self).__init__()
self.conv = nn.Conv2d(input_channel, output_channel, self.conv = nn.Conv2d(input_channel, output_channel,
kernel_size=1, stride=1, padding=0, has_bias=False, pad_mode="same") kernel_size=1, stride=1, padding=0, has_bias=False, pad_mode="same")
self.bn = nn.BatchNorm2d(output_channel, momentum=0.1, eps=0.0001) self.bn = nn.BatchNorm2d(output_channel, momentum=0.1, eps=0.0001)
self.relu = P.ReLU() self.relu = P.ReLU()
self.mean = P.ReduceMean(keep_dims=True) self.mean = P.ReduceMean(keep_dims=True)
@ -118,7 +117,7 @@ def test_conv_bn_fusion():
def __init__(self): def __init__(self):
super(Net, self).__init__() super(Net, self).__init__()
self.conv = nn.Conv2d(input_channel, output_channel, self.conv = nn.Conv2d(input_channel, output_channel,
kernel_size=1, stride=1, padding=0, has_bias=False, pad_mode="same") kernel_size=1, stride=1, padding=0, has_bias=False, pad_mode="same")
self.bn = nn.BatchNorm2d(output_channel, momentum=0.1, eps=0.0001) self.bn = nn.BatchNorm2d(output_channel, momentum=0.1, eps=0.0001)
self.mean = P.ReduceMean(keep_dims=True) self.mean = P.ReduceMean(keep_dims=True)
self.reshape = P.Reshape() self.reshape = P.Reshape()

15
tests/st/fusion/test_tbe_eltwise_fusion_1.py
Unescape View File

@ -13,16 +13,15 @@
# limitations under the License. # limitations under the License.
# ============================================================================ # ============================================================================
import pytest import pytest
from mindspore import Tensor
from mindspore.ops import operations as P
import mindspore.nn as nn
from mindspore.common.api import ms_function
import mindspore.common.dtype as mstype
import numpy as np import numpy as np
import mindspore.context as context import mindspore.context as context
from mindspore.common.initializer import initializer import mindspore.nn as nn
from mindspore.common.parameter import Parameter from mindspore import Tensor
from mindspore.ops import operations as P
context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") context.set_context(mode=context.GRAPH_MODE, device_target="Ascend")
class Net(nn.Cell): class Net(nn.Cell):
def __init__(self): def __init__(self):
super(Net, self).__init__() super(Net, self).__init__()
@ -35,6 +34,7 @@ class Net(nn.Cell):
x = self.relu(x) x = self.relu(x)
return x return x
@pytest.mark.level0 @pytest.mark.level0
@pytest.mark.platform_arm_ascend_training @pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training @pytest.mark.platform_x86_ascend_training
@ -43,5 +43,4 @@ def test_net():
x = np.random.randn(32, 10).astype(np.float32) x = np.random.randn(32, 10).astype(np.float32)
relu_relu = Net() relu_relu = Net()
output = relu_relu(Tensor(x)) output = relu_relu(Tensor(x))
print(x)
print(output.asnumpy()) print(output.asnumpy())

17
tests/st/fusion/test_tbe_eltwise_fusion_2.py
Unescape View File

@ -13,16 +13,15 @@
# limitations under the License. # limitations under the License.
# ============================================================================ # ============================================================================
import pytest import pytest
from mindspore import Tensor
from mindspore.ops import operations as P
import mindspore.nn as nn
from mindspore.common.api import ms_function
import mindspore.common.dtype as mstype
import numpy as np import numpy as np
import mindspore.context as context import mindspore.context as context
from mindspore.common.initializer import initializer import mindspore.nn as nn
from mindspore.common.parameter import Parameter from mindspore import Tensor
from mindspore.ops import operations as P
context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") context.set_context(mode=context.GRAPH_MODE, device_target="Ascend")
class Net(nn.Cell): class Net(nn.Cell):
def __init__(self): def __init__(self):
super(Net, self).__init__() super(Net, self).__init__()
@ -41,6 +40,7 @@ class Net(nn.Cell):
x = self.relu(x) x = self.relu(x)
return x return x
@pytest.mark.level0 @pytest.mark.level0
@pytest.mark.platform_arm_ascend_training @pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training @pytest.mark.platform_x86_ascend_training
@ -50,5 +50,4 @@ def test_net():
y = np.random.randn(10).astype(np.float32) y = np.random.randn(10).astype(np.float32)
net = Net() net = Net()
output = net(Tensor(x), Tensor(y)) output = net(Tensor(x), Tensor(y))
print(x) print(output.asnumpy())
print(output.asnumpy())

14
tests/st/fusion/test_tbe_multi_inout_eltwise_fusion.py
Unescape View File

@ -12,15 +12,14 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
# ============================================================================ # ============================================================================
from mindspore import Tensor
from mindspore.ops import operations as P
import mindspore.nn as nn
import mindspore.common.dtype as mstype
import numpy as np import numpy as np
import mindspore.context as context import mindspore.context as context
from mindspore.common.parameter import Parameter import mindspore.nn as nn
from mindspore import Tensor
from mindspore.ops import operations as P
context.set_context(mode=context.GRAPH_MODE, device_id=4, device_target="Ascend") context.set_context(mode=context.GRAPH_MODE, device_id=4, device_target="Ascend")
#context.set_context(enable_task_sink=True)
class Net(nn.Cell): class Net(nn.Cell):
def __init__(self): def __init__(self):
@ -39,6 +38,7 @@ class Net(nn.Cell):
z = self.add(z1, z2) z = self.add(z1, z2)
return z return z
def test_net(): def test_net():
x = np.random.randn(32, 10).astype(np.float32) x = np.random.randn(32, 10).astype(np.float32)
y = np.random.randn(32, 10).astype(np.float32) y = np.random.randn(32, 10).astype(np.float32)
@ -46,6 +46,4 @@ def test_net():
h = np.random.randn(10).astype(np.float32) h = np.random.randn(10).astype(np.float32)
relu_relu = Net() relu_relu = Net()
output = relu_relu(Tensor(x), Tensor(y), Tensor(k), Tensor(h)) output = relu_relu(Tensor(x), Tensor(y), Tensor(k), Tensor(h))
print(x)
print(output.asnumpy()) print(output.asnumpy())

17
tests/st/fusion/test_tbe_reduce_eltwise_fusion.py
Unescape View File

@ -13,17 +13,16 @@
# limitations under the License. # limitations under the License.
# ============================================================================ # ============================================================================
import pytest import pytest
import numpy as np
import mindspore.context as context
import mindspore.nn as nn
from mindspore import Tensor from mindspore import Tensor
from mindspore.ops import operations as P from mindspore.ops import operations as P
from mindspore.ops.operations import _grad_ops as G from mindspore.ops.operations import _grad_ops as G
import mindspore.nn as nn
from mindspore.common.api import ms_function
import mindspore.common.dtype as mstype
import numpy as np
import mindspore.context as context
from mindspore.common.initializer import initializer
from mindspore.common.parameter import Parameter
context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") context.set_context(mode=context.GRAPH_MODE, device_target="Ascend")
class Net(nn.Cell): class Net(nn.Cell):
def __init__(self): def __init__(self):
super(Net, self).__init__() super(Net, self).__init__()
@ -41,6 +40,7 @@ class Net(nn.Cell):
x = self.relu(x) x = self.relu(x)
return x return x
@pytest.mark.level0 @pytest.mark.level0
@pytest.mark.platform_arm_ascend_training @pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training @pytest.mark.platform_x86_ascend_training
@ -49,5 +49,4 @@ def test_net():
x = np.random.randn(32, 10).astype(np.float32) x = np.random.randn(32, 10).astype(np.float32)
net = Net() net = Net()
output = net(Tensor(x)) output = net(Tensor(x))
print(x) print(output.asnumpy())
print(output.asnumpy())

3
tests/st/mem_reuse/check_file.py
Unescape View File

@ -14,6 +14,7 @@
# ============================================================================ # ============================================================================
import os import os
import filecmp import filecmp
curr_path = os.path.abspath(os.curdir) curr_path = os.path.abspath(os.curdir)
file_memreuse = curr_path + "/mem_reuse_check/memreuse.ir" file_memreuse = curr_path + "/mem_reuse_check/memreuse.ir"
file_normal = curr_path + "/mem_reuse_check/normal_mem.ir" file_normal = curr_path + "/mem_reuse_check/normal_mem.ir"
@ -23,5 +24,3 @@ checker = os.path.exists(file_normal)
assert (checker, True) assert (checker, True)
checker = filecmp.cmp(file_memreuse, file_normal) checker = filecmp.cmp(file_memreuse, file_normal)
assert (checker, True) assert (checker, True)

2
tests/st/mem_reuse/resnet.py
Unescape View File

@ -19,6 +19,7 @@ from mindspore.ops import operations as P
from mindspore.common.initializer import initializer from mindspore.common.initializer import initializer
from mindspore.common import dtype as mstype from mindspore.common import dtype as mstype
def weight_variable(shape): def weight_variable(shape):
return initializer('XavierUniform', shape=shape, dtype=mstype.float32) return initializer('XavierUniform', shape=shape, dtype=mstype.float32)
@ -297,4 +298,3 @@ class ResNet(nn.Cell):
def resnet50(batch_size, num_classes): def resnet50(batch_size, num_classes):
return ResNet(ResidualBlock, [3, 4, 6, 3], num_classes, batch_size) return ResNet(ResidualBlock, [3, 4, 6, 3], num_classes, batch_size)

25
tests/st/mem_reuse/resnet_cifar_memreuse.py
Unescape View File

@ -12,16 +12,17 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
# ============================================================================ # ============================================================================
import argparse
import os
import numpy as np
import mindspore.context as context
import mindspore.nn as nn import mindspore.nn as nn
import mindspore.common.dtype as mstype
from mindspore import Tensor from mindspore import Tensor
from mindspore.ops import operations as P from mindspore.ops import operations as P
from mindspore.ops import functional as F
from mindspore.nn.optim.momentum import Momentum from mindspore.nn.optim.momentum import Momentum
from mindspore.train.model import Model, ParallelMode from mindspore.train.model import Model, ParallelMode
from mindspore import context
import mindspore.common.dtype as mstype
import os
import numpy as np
import mindspore.ops.functional as F
from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor
from mindspore.train.serialization import load_checkpoint, load_param_into_net from mindspore.train.serialization import load_checkpoint, load_param_into_net
import mindspore.dataset as de import mindspore.dataset as de
@ -30,11 +31,11 @@ import mindspore.dataset.transforms.vision.c_transforms as vision
from mindspore.communication.management import init from mindspore.communication.management import init
from resnet import resnet50 from resnet import resnet50
import random import random
random.seed(1) random.seed(1)
np.random.seed(1) np.random.seed(1)
de.config.set_seed(1) de.config.set_seed(1)
import argparse
parser = argparse.ArgumentParser(description='Image classification') parser = argparse.ArgumentParser(description='Image classification')
parser.add_argument('--run_distribute', type=bool, default=False, help='Run distribute') parser.add_argument('--run_distribute', type=bool, default=False, help='Run distribute')
parser.add_argument('--device_num', type=int, default=1, help='Device num.') parser.add_argument('--device_num', type=int, default=1, help='Device num.')
@ -47,9 +48,9 @@ parser.add_argument('--checkpoint_path', type=str, default=None, help='Checkpoin
parser.add_argument('--dataset_path', type=str, default="/var/log/npu/datasets/cifar", help='Dataset path') parser.add_argument('--dataset_path', type=str, default="/var/log/npu/datasets/cifar", help='Dataset path')
args_opt = parser.parse_args() args_opt = parser.parse_args()
device_id=int(os.getenv('DEVICE_ID')) device_id = int(os.getenv('DEVICE_ID'))
data_home=args_opt.dataset_path data_home = args_opt.dataset_path
context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") context.set_context(mode=context.GRAPH_MODE, device_target="Ascend")
context.set_context(enable_task_sink=True, device_id=device_id) context.set_context(enable_task_sink=True, device_id=device_id)
@ -64,8 +65,8 @@ def create_dataset(repeat_num=1, training=True):
ds = de.Cifar10Dataset(data_dir) ds = de.Cifar10Dataset(data_dir)
if args_opt.run_distribute: if args_opt.run_distribute:
rank_id=int(os.getenv('RANK_ID')) rank_id = int(os.getenv('RANK_ID'))
rank_size=int(os.getenv('RANK_SIZE')) rank_size = int(os.getenv('RANK_SIZE'))
ds = de.Cifar10Dataset(data_dir, num_shards=rank_size, shard_id=rank_id) ds = de.Cifar10Dataset(data_dir, num_shards=rank_size, shard_id=rank_id)
resize_height = 224 resize_height = 224
@ -74,9 +75,9 @@ def create_dataset(repeat_num=1, training=True):
shift = 0.0 shift = 0.0
# define map operations # define map operations
random_crop_op = vision.RandomCrop((32, 32), (4, 4, 4, 4)) # padding_mode default CONSTANT random_crop_op = vision.RandomCrop((32, 32), (4, 4, 4, 4)) # padding_mode default CONSTANT
random_horizontal_op = vision.RandomHorizontalFlip() random_horizontal_op = vision.RandomHorizontalFlip()
resize_op = vision.Resize((resize_height, resize_width)) # interpolation default BILINEAR resize_op = vision.Resize((resize_height, resize_width)) # interpolation default BILINEAR
rescale_op = vision.Rescale(rescale, shift) rescale_op = vision.Rescale(rescale, shift)
normalize_op = vision.Normalize((0.4465, 0.4822, 0.4914), (0.2010, 0.1994, 0.2023)) normalize_op = vision.Normalize((0.4465, 0.4822, 0.4914), (0.2010, 0.1994, 0.2023))
changeswap_op = vision.HWC2CHW() changeswap_op = vision.HWC2CHW()

12
tests/st/mem_reuse/resnet_cifar_normal.py
Unescape View File

@ -12,16 +12,17 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
# ============================================================================ # ============================================================================
import argparse
import os
import numpy as np
import mindspore.context as context
import mindspore.nn as nn import mindspore.nn as nn
import mindspore.common.dtype as mstype
from mindspore import Tensor from mindspore import Tensor
from mindspore.ops import operations as P from mindspore.ops import operations as P
from mindspore.ops import functional as F
from mindspore.nn.optim.momentum import Momentum from mindspore.nn.optim.momentum import Momentum
from mindspore.train.model import Model, ParallelMode from mindspore.train.model import Model, ParallelMode
from mindspore import context
import mindspore.common.dtype as mstype
import os
import numpy as np
import mindspore.ops.functional as F
from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor
from mindspore.train.serialization import load_checkpoint, load_param_into_net from mindspore.train.serialization import load_checkpoint, load_param_into_net
import mindspore.dataset as de import mindspore.dataset as de
@ -35,7 +36,6 @@ random.seed(1)
np.random.seed(1) np.random.seed(1)
de.config.set_seed(1) de.config.set_seed(1)
import argparse
parser = argparse.ArgumentParser(description='Image classification') parser = argparse.ArgumentParser(description='Image classification')
parser.add_argument('--run_distribute', type=bool, default=False, help='Run distribute') parser.add_argument('--run_distribute', type=bool, default=False, help='Run distribute')

4
tests/st/nccl/test_nccl_all.py
Unescape View File

@ -15,6 +15,7 @@
import os import os
import pytest import pytest
@pytest.mark.level0 @pytest.mark.level0
@pytest.mark.platform_x86_gpu_training @pytest.mark.platform_x86_gpu_training
@pytest.mark.env_single @pytest.mark.env_single
@ -22,6 +23,7 @@ def test_nccl_lenet():
return_code = os.system("mpirun -n 8 pytest -s test_nccl_lenet.py") return_code = os.system("mpirun -n 8 pytest -s test_nccl_lenet.py")
assert(return_code == 0) assert(return_code == 0)
@pytest.mark.level0 @pytest.mark.level0
@pytest.mark.platform_x86_gpu_training @pytest.mark.platform_x86_gpu_training
@pytest.mark.env_single @pytest.mark.env_single
@ -29,6 +31,7 @@ def test_nccl_all_reduce_op():
return_code = os.system("mpirun -n 8 pytest -s test_nccl_all_reduce_op.py") return_code = os.system("mpirun -n 8 pytest -s test_nccl_all_reduce_op.py")
assert(return_code == 0) assert(return_code == 0)
@pytest.mark.level0 @pytest.mark.level0
@pytest.mark.platform_x86_gpu_training @pytest.mark.platform_x86_gpu_training
@pytest.mark.env_single @pytest.mark.env_single
@ -36,6 +39,7 @@ def test_nccl_all_gather_op():
return_code = os.system("mpirun -n 8 pytest -s test_nccl_all_gather_op.py") return_code = os.system("mpirun -n 8 pytest -s test_nccl_all_gather_op.py")
assert(return_code == 0) assert(return_code == 0)
@pytest.mark.level0 @pytest.mark.level0
@pytest.mark.platform_x86_gpu_training @pytest.mark.platform_x86_gpu_training
@pytest.mark.env_single @pytest.mark.env_single

13
tests/st/nccl/test_nccl_all_gather_op.py
Unescape View File

@ -12,23 +12,25 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
# ============================================================================ # ============================================================================
from mindspore import Tensor
from mindspore.ops import operations as P
import mindspore.nn as nn
import numpy as np import numpy as np
import mindspore.context as context import mindspore.context as context
import mindspore.nn as nn
from mindspore import Tensor
from mindspore.ops import operations as P
from mindspore.common.initializer import initializer from mindspore.common.initializer import initializer
from mindspore.common.parameter import Parameter from mindspore.common.parameter import Parameter
from mindspore.communication.management import init, NCCL_WORLD_COMM_GROUP, get_rank, get_group_size from mindspore.communication.management import init, NCCL_WORLD_COMM_GROUP, get_rank, get_group_size
context.set_context(mode=context.GRAPH_MODE, device_target='GPU') context.set_context(mode=context.GRAPH_MODE, device_target='GPU')
init('nccl') init('nccl')
rank = get_rank() rank = get_rank()
size = get_group_size() size = get_group_size()
x = np.ones([1,1,3,3]).astype(np.float32) * 0.01 * (rank + 1) x = np.ones([1, 1, 3, 3]).astype(np.float32) * 0.01 * (rank + 1)
class Net(nn.Cell): class Net(nn.Cell):
def __init__( self): def __init__(self):
super(Net, self).__init__() super(Net, self).__init__()
self.all_gather = P.AllGather(group=NCCL_WORLD_COMM_GROUP) self.all_gather = P.AllGather(group=NCCL_WORLD_COMM_GROUP)
self.x = Parameter(initializer(Tensor(x), x.shape), name='x') self.x = Parameter(initializer(Tensor(x), x.shape), name='x')
@ -36,6 +38,7 @@ class Net(nn.Cell):
def construct(self): def construct(self):
return self.all_gather(self.x) return self.all_gather(self.x)
def test_AllGather(): def test_AllGather():
all_gather = Net() all_gather = Net()
output = all_gather() output = all_gather()

19
tests/st/nccl/test_nccl_all_reduce_op.py
Unescape View File

@ -12,23 +12,25 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
# ============================================================================ # ============================================================================
from mindspore import Tensor
from mindspore.ops import operations as P
import mindspore.nn as nn
import numpy as np import numpy as np
import mindspore.context as context import mindspore.context as context
import mindspore.nn as nn
from mindspore import Tensor
from mindspore.ops import operations as P
from mindspore.common.initializer import initializer from mindspore.common.initializer import initializer
from mindspore.common.parameter import Parameter from mindspore.common.parameter import Parameter
from mindspore.communication.management import init, NCCL_WORLD_COMM_GROUP, get_rank, get_group_size from mindspore.communication.management import init, NCCL_WORLD_COMM_GROUP, get_rank, get_group_size
context.set_context(mode=context.GRAPH_MODE, device_target='GPU') context.set_context(mode=context.GRAPH_MODE, device_target='GPU')
init('nccl') init('nccl')
rank = get_rank() rank = get_rank()
size = get_group_size() size = get_group_size()
x = np.ones([3,1,3,3]).astype(np.float32) * 0.01 * (rank + 1) x = np.ones([3, 1, 3, 3]).astype(np.float32) * 0.01 * (rank + 1)
class Net(nn.Cell): class Net(nn.Cell):
def __init__( self): def __init__(self):
super(Net, self).__init__() super(Net, self).__init__()
self.x1 = Parameter(initializer(Tensor(x), x.shape), name='x1') self.x1 = Parameter(initializer(Tensor(x), x.shape), name='x1')
self.x2 = Parameter(initializer(Tensor(x), x.shape), name='x2') self.x2 = Parameter(initializer(Tensor(x), x.shape), name='x2')
@ -47,6 +49,7 @@ class Net(nn.Cell):
self.all_reduce2(self.x2), self.all_reduce2(self.x2),
self.all_reduce3(self.x3)) self.all_reduce3(self.x3))
def test_AllReduce(): def test_AllReduce():
all_reduce = Net() all_reduce = Net()
output = all_reduce() output = all_reduce()
@ -58,16 +61,16 @@ def test_AllReduce():
diff0 = output[0].asnumpy() - expect0 diff0 = output[0].asnumpy() - expect0
error0 = np.ones(shape=expect0.shape) * 1.0e-5 error0 = np.ones(shape=expect0.shape) * 1.0e-5
assert np.all(diff0 < error0) assert np.all(diff0 < error0)
assert (output[0].shape() == expect0.shape) assert output[0].shape() == expect0.shape
expect1 = expect0 expect1 = expect0
diff1 = output[1].asnumpy() - expect1 diff1 = output[1].asnumpy() - expect1
error1 = np.ones(shape=expect1.shape) * 1.0e-5 error1 = np.ones(shape=expect1.shape) * 1.0e-5
assert np.all(diff1 < error1) assert np.all(diff1 < error1)
assert (output[1].shape() == expect1.shape) assert output[1].shape() == expect1.shape
expect2 = expect1 expect2 = expect1
diff2 = output[2].asnumpy() - expect2 diff2 = output[2].asnumpy() - expect2
error2 = np.ones(shape=expect2.shape) * 1.0e-5 error2 = np.ones(shape=expect2.shape) * 1.0e-5
assert np.all(diff2 < error2) assert np.all(diff2 < error2)
assert (output[2].shape() == expect2.shape) assert output[2].shape() == expect2.shape

23
tests/st/nccl/test_nccl_lenet.py
Unescape View File

@ -12,16 +12,15 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
# ============================================================================ # ============================================================================
import numpy as np
from mindspore.nn import Dense
import mindspore.nn as nn
import datetime import datetime
import numpy as np
import mindspore.context as context import mindspore.context as context
from mindspore.communication.management import init, NCCL_WORLD_COMM_GROUP, get_rank, get_group_size import mindspore.nn as nn
from mindspore import Tensor
from mindspore.nn.optim import Momentum from mindspore.nn.optim import Momentum
from mindspore.nn import TrainOneStepCell, WithLossCell from mindspore.nn import TrainOneStepCell, WithLossCell
from mindspore.ops import operations as P from mindspore.ops import operations as P
from mindspore.common.tensor import Tensor from mindspore.communication.management import init, get_rank, get_group_size
context.set_context(mode=context.GRAPH_MODE, device_target="GPU") context.set_context(mode=context.GRAPH_MODE, device_target="GPU")
init('nccl') init('nccl')
@ -31,6 +30,7 @@ total = 5000
batch_size = 32 batch_size = 32
mini_batch = total // batch_size mini_batch = total // batch_size
class LeNet(nn.Cell): class LeNet(nn.Cell):
def __init__(self): def __init__(self):
super(LeNet, self).__init__() super(LeNet, self).__init__()
@ -43,15 +43,15 @@ class LeNet(nn.Cell):
self.conv2 = nn.Conv2d(6, 16, (5, 5), weight_init=weight2, pad_mode='valid', stride=1, padding=0) self.conv2 = nn.Conv2d(6, 16, (5, 5), weight_init=weight2, pad_mode='valid', stride=1, padding=0)
self.pool = nn.MaxPool2d(kernel_size=2, stride=2, pad_mode="valid") self.pool = nn.MaxPool2d(kernel_size=2, stride=2, pad_mode="valid")
self.reshape = P.Reshape() self.reshape = P.Reshape()
weight1 = Tensor(np.ones([120, 400]).astype(np.float32) * 0.01) weight1 = Tensor(np.ones([120, 400]).astype(np.float32) * 0.01)
self.fc1 = Dense(400, 120, weight_init=weight1) self.fc1 = nn.Dense(400, 120, weight_init=weight1)
weight2 = Tensor(np.ones([84, 120]).astype(np.float32) * 0.01) weight2 = Tensor(np.ones([84, 120]).astype(np.float32) * 0.01)
self.fc2 = Dense(120, 84, weight_init=weight2) self.fc2 = nn.Dense(120, 84, weight_init=weight2)
weight3 = Tensor(np.ones([10, 84]).astype(np.float32) * 0.01) weight3 = Tensor(np.ones([10, 84]).astype(np.float32) * 0.01)
self.fc3 = Dense(84, 10, weight_init=weight3) self.fc3 = nn.Dense(84, 10, weight_init=weight3)
def construct(self, input_x): def construct(self, input_x):
output = self.conv1(input_x) output = self.conv1(input_x)
@ -66,6 +66,7 @@ class LeNet(nn.Cell):
output = self.fc3(output) output = self.fc3(output)
return output return output
def test_lenet_nccl(): def test_lenet_nccl():
net = LeNet() net = LeNet()
net.set_train() net.set_train()

12
tests/st/nccl/test_nccl_reduce_scatter_op.py
Unescape View File

@ -12,11 +12,11 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
# ============================================================================ # ============================================================================
from mindspore import Tensor
from mindspore.ops import operations as P
import mindspore.nn as nn
import numpy as np import numpy as np
import mindspore.context as context import mindspore.context as context
import mindspore.nn as nn
from mindspore import Tensor
from mindspore.ops import operations as P
from mindspore.common.initializer import initializer from mindspore.common.initializer import initializer
from mindspore.common.parameter import Parameter from mindspore.common.parameter import Parameter
from mindspore.communication.management import init, NCCL_WORLD_COMM_GROUP, get_rank, get_group_size from mindspore.communication.management import init, NCCL_WORLD_COMM_GROUP, get_rank, get_group_size
@ -27,8 +27,9 @@ rank = get_rank()
size = get_group_size() size = get_group_size()
x = np.ones([size, 1, 3, 3]).astype(np.float32) * 0.01 * (rank + 1) x = np.ones([size, 1, 3, 3]).astype(np.float32) * 0.01 * (rank + 1)
class Net(nn.Cell): class Net(nn.Cell):
def __init__( self): def __init__(self):
super(Net, self).__init__() super(Net, self).__init__()
self.x = Parameter(initializer(Tensor(x), x.shape), name='x') self.x = Parameter(initializer(Tensor(x), x.shape), name='x')
@ -46,6 +47,7 @@ class Net(nn.Cell):
self.reduce_scatter2(self.x), self.reduce_scatter2(self.x),
self.reduce_scatter3(self.x)) self.reduce_scatter3(self.x))
def test_ReduceScatter(): def test_ReduceScatter():
reduce_scatter = Net() reduce_scatter = Net()
output = reduce_scatter() output = reduce_scatter()
@ -53,7 +55,7 @@ def test_ReduceScatter():
sum = np.ones([size, 1, 3, 3]).astype(np.float32) * 0 sum = np.ones([size, 1, 3, 3]).astype(np.float32) * 0
for i in range(size): for i in range(size):
sum += np.ones([size, 1, 3, 3]).astype(np.float32) * 0.01 * (i + 1) sum += np.ones([size, 1, 3, 3]).astype(np.float32) * 0.01 * (i + 1)
expect0 = sum[rank : rank + 1] expect0 = sum[rank: rank + 1]
diff0 = output[0].asnumpy() - expect0 diff0 = output[0].asnumpy() - expect0
error0 = np.ones(shape=expect0.shape) * 1.0e-5 error0 = np.ones(shape=expect0.shape) * 1.0e-5
assert np.all(diff0 < error0) assert np.all(diff0 < error0)

1
tests/st/networks/models/alexnet.py
Unescape View File

@ -16,6 +16,7 @@ import mindspore.nn as nn
from mindspore.ops import operations as P from mindspore.ops import operations as P
from mindspore.nn import Dense from mindspore.nn import Dense
class AlexNet(nn.Cell): class AlexNet(nn.Cell):
def __init__(self, num_classes=10): def __init__(self, num_classes=10):
super(AlexNet, self).__init__() super(AlexNet, self).__init__()

16
tests/st/networks/models/bert/bert_tdt_no_lossscale.py
Unescape View File

@ -18,21 +18,22 @@
import os import os
import pytest import pytest
import numpy as np import numpy as np
from numpy import allclose import mindspore.context as context
import mindspore.common.dtype as mstype import mindspore.common.dtype as mstype
import mindspore.dataset.engine.datasets as de import mindspore.dataset.engine.datasets as de
import mindspore.dataset.transforms.c_transforms as C import mindspore.dataset.transforms.c_transforms as C
from mindspore import context from mindspore import Tensor
from mindspore.common.tensor import Tensor
from mindspore.train.model import Model from mindspore.train.model import Model
from mindspore.train.callback import Callback from mindspore.train.callback import Callback
from mindspore.model_zoo.Bert_NEZHA import BertConfig, BertNetworkWithLoss, BertTrainOneStepCell from mindspore.model_zoo.Bert_NEZHA import BertConfig, BertNetworkWithLoss, BertTrainOneStepCell
from mindspore.nn.optim import Momentum from mindspore.nn.optim import Momentum
from mindspore import log as logger from mindspore import log as logger
_current_dir = os.path.dirname(os.path.realpath(__file__)) _current_dir = os.path.dirname(os.path.realpath(__file__))
DATA_DIR = ["/home/workspace/mindspore_dataset/bert/example/examples.tfrecord"] DATA_DIR = ["/home/workspace/mindspore_dataset/bert/example/examples.tfrecord"]
SCHEMA_DIR = "/home/workspace/mindspore_dataset/bert/example/datasetSchema.json" SCHEMA_DIR = "/home/workspace/mindspore_dataset/bert/example/datasetSchema.json"
def get_config(version='base', batch_size=1): def get_config(version='base', batch_size=1):
"""get config""" """get config"""
if version == 'base': if version == 'base':
@ -99,13 +100,14 @@ def get_config(version='base', batch_size=1):
bert_config = BertConfig(batch_size=batch_size) bert_config = BertConfig(batch_size=batch_size)
return bert_config return bert_config
def me_de_train_dataset(): def me_de_train_dataset():
"""test me de train dataset""" """test me de train dataset"""
# apply repeat operations # apply repeat operations
repeat_count = 1 repeat_count = 1
ds = de.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["input_ids", "input_mask", "segment_ids", ds = de.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["input_ids", "input_mask", "segment_ids",
"next_sentence_labels", "masked_lm_positions", "next_sentence_labels", "masked_lm_positions",
"masked_lm_ids", "masked_lm_weights"], shuffle=False) "masked_lm_ids", "masked_lm_weights"], shuffle=False)
type_cast_op = C.TypeCast(mstype.int32) type_cast_op = C.TypeCast(mstype.int32)
ds = ds.map(input_columns="masked_lm_ids", operations=type_cast_op) ds = ds.map(input_columns="masked_lm_ids", operations=type_cast_op)
ds = ds.map(input_columns="masked_lm_positions", operations=type_cast_op) ds = ds.map(input_columns="masked_lm_positions", operations=type_cast_op)
@ -137,6 +139,7 @@ class ModelCallback(Callback):
self.loss_list.append(cb_params.net_outputs.asnumpy()[0]) self.loss_list.append(cb_params.net_outputs.asnumpy()[0])
logger.info("epoch: {}, outputs are {}".format(cb_params.cur_epoch_num, str(cb_params.net_outputs))) logger.info("epoch: {}, outputs are {}".format(cb_params.cur_epoch_num, str(cb_params.net_outputs)))
@pytest.mark.level0 @pytest.mark.level0
@pytest.mark.platform_arm_ascend_training @pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training @pytest.mark.platform_x86_ascend_training
@ -180,7 +183,8 @@ def test_bert_tdt():
expect_out = [12.19179, 11.965041, 11.969687, 11.97815, 11.969171, 12.603289, 12.165594, expect_out = [12.19179, 11.965041, 11.969687, 11.97815, 11.969171, 12.603289, 12.165594,
12.824818, 12.38842, 12.604046] 12.824818, 12.38842, 12.604046]
logger.info("expected loss value output: {}".format(expect_out)) logger.info("expected loss value output: {}".format(expect_out))
assert allclose(loss_value, expect_out, 0.00001, 0.00001) assert np.allclose(loss_value, expect_out, 0.00001, 0.00001)
if __name__ == '__main__': if __name__ == '__main__':
test_bert_tdt() test_bert_tdt()

3
tests/st/networks/models/lenet.py
Unescape View File

@ -14,9 +14,10 @@
# ============================================================================ # ============================================================================
import numpy as np import numpy as np
import mindspore.nn as nn import mindspore.nn as nn
from mindspore import Tensor
from mindspore.ops import operations as P from mindspore.ops import operations as P
from mindspore.nn import Dense from mindspore.nn import Dense
from mindspore import Tensor
class LeNet(nn.Cell): class LeNet(nn.Cell):
def __init__(self): def __init__(self):

17
tests/st/networks/models/resnetv1_5.py
Unescape View File

@ -13,9 +13,10 @@
# limitations under the License. # limitations under the License.
# ============================================================================ # ============================================================================
import numpy as np import numpy as np
from mindspore.common.tensor import Tensor
import mindspore.nn as nn import mindspore.nn as nn
import mindspore.ops.operations as P from mindspore import Tensor
from mindspore.ops import operations as P
def weight_variable(shape): def weight_variable(shape):
ones = np.ones(shape).astype(np.float32) ones = np.ones(shape).astype(np.float32)
@ -37,7 +38,7 @@ def conv3x3(in_channels, out_channels, stride=1, padding=0):
weight_shape = (out_channels, in_channels, 3, 3) weight_shape = (out_channels, in_channels, 3, 3)
weight = weight_variable(weight_shape) weight = weight_variable(weight_shape)
return nn.Conv2d(in_channels, out_channels, return nn.Conv2d(in_channels, out_channels,
kernel_size=3, stride=stride, padding=padding, weight_init=weight, has_bias=False, pad_mode="same") kernel_size=3, stride=stride, padding=padding, weight_init=weight, has_bias=False, pad_mode="same")
def conv1x1(in_channels, out_channels, stride=1, padding=0): def conv1x1(in_channels, out_channels, stride=1, padding=0):
@ -45,7 +46,7 @@ def conv1x1(in_channels, out_channels, stride=1, padding=0):
weight_shape = (out_channels, in_channels, 1, 1) weight_shape = (out_channels, in_channels, 1, 1)
weight = weight_variable(weight_shape) weight = weight_variable(weight_shape)
return nn.Conv2d(in_channels, out_channels, return nn.Conv2d(in_channels, out_channels,
kernel_size=1, stride=stride, padding=padding, weight_init=weight, has_bias=False, pad_mode="same") kernel_size=1, stride=stride, padding=padding, weight_init=weight, has_bias=False, pad_mode="same")
def conv7x7(in_channels, out_channels, stride=1, padding=0): def conv7x7(in_channels, out_channels, stride=1, padding=0):
@ -53,7 +54,7 @@ def conv7x7(in_channels, out_channels, stride=1, padding=0):
weight_shape = (out_channels, in_channels, 7, 7) weight_shape = (out_channels, in_channels, 7, 7)
weight = weight_variable(weight_shape) weight = weight_variable(weight_shape)
return nn.Conv2d(in_channels, out_channels, return nn.Conv2d(in_channels, out_channels,
kernel_size=7, stride=stride, padding=padding, weight_init=weight, has_bias=False, pad_mode="same") kernel_size=7, stride=stride, padding=padding, weight_init=weight, has_bias=False, pad_mode="same")
def bn_with_initialize(out_channels): def bn_with_initialize(out_channels):
@ -63,7 +64,7 @@ def bn_with_initialize(out_channels):
beta = weight_variable_0(shape) beta = weight_variable_0(shape)
gamma = weight_variable_1(shape) gamma = weight_variable_1(shape)
bn = nn.BatchNorm2d(out_channels, momentum=0.1, eps=0.0001, gamma_init=gamma, bn = nn.BatchNorm2d(out_channels, momentum=0.1, eps=0.0001, gamma_init=gamma,
beta_init=beta, moving_mean_init=mean, moving_var_init=var) beta_init=beta, moving_mean_init=mean, moving_var_init=var)
return bn return bn
@ -74,7 +75,7 @@ def bn_with_initialize_last(out_channels):
beta = weight_variable_0(shape) beta = weight_variable_0(shape)
gamma = weight_variable_0(shape) gamma = weight_variable_0(shape)
bn = nn.BatchNorm2d(out_channels, momentum=0.1, eps=0.0001, gamma_init=gamma, bn = nn.BatchNorm2d(out_channels, momentum=0.1, eps=0.0001, gamma_init=gamma,
beta_init=beta, moving_mean_init=mean, moving_var_init=var) beta_init=beta, moving_mean_init=mean, moving_var_init=var)
return bn return bn
@ -294,6 +295,6 @@ class ResNet(nn.Cell):
x = self.fc(x) x = self.fc(x)
return x return x
def resnet50(batch_size, num_classes): def resnet50(batch_size, num_classes):
return ResNet(ResidualBlock, [3, 4, 6, 3], num_classes, batch_size) return ResNet(ResidualBlock, [3, 4, 6, 3], num_classes, batch_size)

Some files were not shown because too many files have changed in this diff Show More

Write Preview
Loading…
Cancel
Save