m53297601
/
mindspore
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

fix and remove useless import of example, st, ut

Browse Source
pull/915/head
xiefangqi 5 years ago committed by chang zherui
parent d4dead939b
commit 8d554d4322
7 changed files with 37 additions and 42 deletions
Show Stats Download Patch File Download Diff File

4
example/Bert_NEZHA_cnwiki/train.py
Unescape View File

@ -36,7 +36,7 @@ import os
import numpy as np
from config import bert_train_cfg, bert_net_cfg
import mindspore.dataset.engine.datasets as de
import mindspore._c_dataengine as deMap
import mindspore.dataset.transforms.c_transforms as C
from mindspore import context
from mindspore.common.tensor import Tensor
from mindspore.train.model import Model
@ -52,7 +52,7 @@ def create_train_dataset(batch_size):
ds = de.StorageDataset([bert_train_cfg.DATA_DIR], bert_train_cfg.SCHEMA_DIR,
columns_list=["input_ids", "input_mask", "segment_ids", "next_sentence_labels",
"masked_lm_positions", "masked_lm_ids", "masked_lm_weights"])
type_cast_op = deMap.TypeCastOp("int32")
type_cast_op = C.TypeCast(mstype.int32)
ds = ds.map(input_columns="masked_lm_ids", operations=type_cast_op)
ds = ds.map(input_columns="masked_lm_positions", operations=type_cast_op)
ds = ds.map(input_columns="next_sentence_labels", operations=type_cast_op)

3
tests/st/mem_reuse/resnet_cifar_memreuse.py
Unescape View File

@ -24,8 +24,7 @@ import numpy as np
import mindspore.ops.functional as F
from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor
from mindspore.train.serialization import load_checkpoint, load_param_into_net
import mindspore.dataengine as de
import mindspore._c_dataengine as deMap
import mindspore.dataset as de
import mindspore.dataset.transforms.c_transforms as C
import mindspore.dataset.transforms.vision.c_transforms as vision
from mindspore.communication.management import init

3
tests/st/mem_reuse/resnet_cifar_normal.py
Unescape View File

@ -24,8 +24,7 @@ import numpy as np
import mindspore.ops.functional as F
from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor
from mindspore.train.serialization import load_checkpoint, load_param_into_net
import mindspore.dataengine as de
import mindspore._c_dataengine as deMap
import mindspore.dataset as de
import mindspore.dataset.transforms.c_transforms as C
import mindspore.dataset.transforms.vision.c_transforms as vision
from mindspore.communication.management import init

4
tests/st/networks/models/bert/bert_tdt_no_lossscale.py
Unescape View File

@ -21,7 +21,7 @@ import numpy as np
from numpy import allclose
import mindspore.common.dtype as mstype
import mindspore.dataset.engine.datasets as de
import mindspore._c_dataengine as deMap
import mindspore.dataset.transforms.c_transforms as C
from mindspore import context
from mindspore.common.tensor import Tensor
from mindspore.train.model import Model
@ -106,7 +106,7 @@ def me_de_train_dataset():
ds = de.StorageDataset(DATA_DIR, SCHEMA_DIR, columns_list=["input_ids", "input_mask", "segment_ids",
"next_sentence_labels", "masked_lm_positions",
"masked_lm_ids", "masked_lm_weights"])
type_cast_op = deMap.TypeCastOp("int32")
type_cast_op = C.TypeCast(mstype.int32)
ds = ds.map(input_columns="masked_lm_ids", operations=type_cast_op)
ds = ds.map(input_columns="masked_lm_positions", operations=type_cast_op)
ds = ds.map(input_columns="next_sentence_labels", operations=type_cast_op)

26
tests/st/ops/davinci/test_tdt_data_ms.py
Unescape View File

@ -12,11 +12,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
import mindspore._c_dataengine as deMap
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as vision
from mindspore.dataset.transforms.vision import Inter
import numpy as np
import sys
from mindspore._c_dataengine import InterpolationMode
import mindspore.context as context
import mindspore.nn as nn
@ -32,7 +32,7 @@ SCHEMA_DIR = "{0}/resnet_all_datasetSchema.json".format(data_path)
def test_me_de_train_dataset():
data_list = ["{0}/train-00001-of-01024.data".format(data_path)]
data_set = ds.StorageDataset(data_list, schema=SCHEMA_DIR,
columns_list=["image/encoded", "image/class/label"])
columns_list=["image/encoded", "image/class/label"])
resize_height = 224
resize_width = 224
@ -41,19 +41,17 @@ def test_me_de_train_dataset():
# define map operations
decode_op = deMap.DecodeOp()
resize_op = deMap.ResizeOp(resize_height, resize_width,
InterpolationMode.DE_INTER_LINEAR) # Bilinear as default
rescale_op = deMap.RescaleOp(rescale, shift)
changemode_op = deMap.ChangeModeOp()
decode_op = vision.Decode()
resize_op = vision.Resize(resize_height, resize_width,
Inter.LINEAR) # Bilinear as default
rescale_op = vision.Rescale(rescale, shift)
# apply map operations on images
data_set = data_set.map(input_column_names="image/encoded", operation=decode_op)
data_set = data_set.map(input_column_names="image/encoded", operation=resize_op)
data_set = data_set.map(input_column_names="image/encoded", operation=rescale_op)
data_set = data_set.map(input_column_names="image/encoded", operation=changemode_op)
changeswap_op = deMap.ChannelSwapOp()
data_set = data_set.map(input_column_names="image/encoded", operation=changeswap_op)
data_set = data_set.map(input_columns="image/encoded", operations=decode_op)
data_set = data_set.map(input_columns="image/encoded", operations=resize_op)
data_set = data_set.map(input_columns="image/encoded", operations=rescale_op)
hwc2chw_op = vision.HWC2CHW()
data_set = data_set.map(input_columns="image/encoded", operations=hwc2chw_op)
data_set = data_set.repeat(1)
# apply batch operations
batch_size = 32

1
tests/ut/python/dataset/test_minddataset.py
Unescape View File

@ -24,7 +24,6 @@ import string
import mindspore.dataset.transforms.vision.c_transforms as vision
import numpy as np
import pytest
from mindspore._c_dataengine import InterpolationMode
from mindspore.dataset.transforms.vision import Inter
from mindspore import log as logger

38
tests/ut/python/dataset/test_project.py
Unescape View File

@ -13,7 +13,8 @@
# limitations under the License.
# ==============================================================================
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore._c_dataengine as de_map
import mindspore.dataset.transforms.c_transforms as C
from mindspore.common import dtype as mstype
from util import ordered_save_and_check
import mindspore.dataset as ds
@ -63,9 +64,8 @@ def test_case_project_map():
data1 = ds.TFRecordDataset(DATA_DIR_TF, SCHEMA_DIR_TF, shuffle=False)
data1 = data1.project(columns=columns)
no_op = de_map.NoOp()
data1 = data1.map(input_columns=["col_3d"], operations=no_op)
type_cast_op = C.TypeCast(mstype.int64)
data1 = data1.map(input_columns=["col_3d"], operations=type_cast_op)
filename = "project_map_after_result.npz"
ordered_save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN)
@ -77,8 +77,8 @@ def test_case_map_project():
data1 = ds.TFRecordDataset(DATA_DIR_TF, SCHEMA_DIR_TF, shuffle=False)
no_op = de_map.NoOp()
data1 = data1.map(input_columns=["col_sint64"], operations=no_op)
type_cast_op = C.TypeCast(mstype.int64)
data1 = data1.map(input_columns=["col_sint64"], operations=type_cast_op)
data1 = data1.project(columns=columns)
@ -92,19 +92,19 @@ def test_case_project_between_maps():
data1 = ds.TFRecordDataset(DATA_DIR_TF, SCHEMA_DIR_TF, shuffle=False)
no_op = de_map.NoOp()
data1 = data1.map(input_columns=["col_3d"], operations=no_op)
data1 = data1.map(input_columns=["col_3d"], operations=no_op)
data1 = data1.map(input_columns=["col_3d"], operations=no_op)
data1 = data1.map(input_columns=["col_3d"], operations=no_op)
type_cast_op = C.TypeCast(mstype.int64)
data1 = data1.map(input_columns=["col_3d"], operations=type_cast_op)
data1 = data1.map(input_columns=["col_3d"], operations=type_cast_op)
data1 = data1.map(input_columns=["col_3d"], operations=type_cast_op)
data1 = data1.map(input_columns=["col_3d"], operations=type_cast_op)
data1 = data1.project(columns=columns)
data1 = data1.map(input_columns=["col_3d"], operations=no_op)
data1 = data1.map(input_columns=["col_3d"], operations=no_op)
data1 = data1.map(input_columns=["col_3d"], operations=no_op)
data1 = data1.map(input_columns=["col_3d"], operations=no_op)
data1 = data1.map(input_columns=["col_3d"], operations=no_op)
data1 = data1.map(input_columns=["col_3d"], operations=type_cast_op)
data1 = data1.map(input_columns=["col_3d"], operations=type_cast_op)
data1 = data1.map(input_columns=["col_3d"], operations=type_cast_op)
data1 = data1.map(input_columns=["col_3d"], operations=type_cast_op)
data1 = data1.map(input_columns=["col_3d"], operations=type_cast_op)
filename = "project_between_maps_result.npz"
ordered_save_and_check(data1, parameters, filename, generate_golden=GENERATE_GOLDEN)
@ -145,12 +145,12 @@ def test_case_map_project_map_project():
data1 = ds.TFRecordDataset(DATA_DIR_TF, SCHEMA_DIR_TF, shuffle=False)
no_op = de_map.NoOp()
data1 = data1.map(input_columns=["col_sint64"], operations=no_op)
type_cast_op = C.TypeCast(mstype.int64)
data1 = data1.map(input_columns=["col_sint64"], operations=type_cast_op)
data1 = data1.project(columns=columns)
data1 = data1.map(input_columns=["col_2d"], operations=no_op)
data1 = data1.map(input_columns=["col_2d"], operations=type_cast_op)
data1 = data1.project(columns=columns)

Write Preview
Loading…
Cancel
Save