diff --git a/mindspore/ccsrc/minddata/dataset/api/python/de_pipeline.cc b/mindspore/ccsrc/minddata/dataset/api/python/de_pipeline.cc index 57207f0676..7d1159a41d 100644 --- a/mindspore/ccsrc/minddata/dataset/api/python/de_pipeline.cc +++ b/mindspore/ccsrc/minddata/dataset/api/python/de_pipeline.cc @@ -733,7 +733,7 @@ Status DEPipeline::ParseMapOp(const py::dict &args, std::shared_ptr * (void)map_builder.SetInColNames(in_col_names); } else if (key == "output_columns") { (void)map_builder.SetOutColNames(ToStringVector(value)); - } else if (key == "columns_order") { + } else if (key == "column_order") { project_columns = ToStringVector(value); } else if (key == "num_parallel_workers") { num_workers = ToInt(value); diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/image_folder_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/image_folder_op.cc index 7959c35371..16168cbce9 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/image_folder_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/image_folder_op.cc @@ -113,7 +113,7 @@ Status ImageFolderOp::PrescanMasterEntry(const std::string &filedir) { num_rows_ = image_label_pairs_.size(); if (num_rows_ == 0) { RETURN_STATUS_UNEXPECTED( - "There is no valid data matching the dataset API ImageFolderDatasetV2.Please check file path or dataset " + "There is no valid data matching the dataset API ImageFolderDataset. Please check file path or dataset " "API validation first."); } // free memory of two queues used for pre-scan diff --git a/mindspore/ccsrc/minddata/dataset/kernels/tensor_op.h b/mindspore/ccsrc/minddata/dataset/kernels/tensor_op.h index 06dba20cd6..1013bd9397 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/tensor_op.h +++ b/mindspore/ccsrc/minddata/dataset/kernels/tensor_op.h @@ -111,7 +111,7 @@ constexpr char kWhitespaceTokenizerOp[] = "WhitespaceTokenizerOp"; constexpr char kWordpieceTokenizerOp[] = "WordpieceTokenizerOp"; constexpr char kRandomChoiceOp[] = "RandomChoiceOp"; constexpr char kRandomApplyOp[] = "RandomApplyOp"; -constexpr char kComposeOp[] = "ComposeOp"; +constexpr char kComposeOp[] = "Compose"; constexpr char kRandomSelectSubpolicyOp[] = "RandomSelectSubpolicyOp"; constexpr char kSentencepieceTokenizerOp[] = "SentencepieceTokenizerOp"; diff --git a/mindspore/dataset/__init__.py b/mindspore/dataset/__init__.py index 8a5cb2367f..0bacb30928 100644 --- a/mindspore/dataset/__init__.py +++ b/mindspore/dataset/__init__.py @@ -19,7 +19,7 @@ can also create samplers with this module to sample data. """ from .core import config -from .engine.datasets import TFRecordDataset, ImageFolderDatasetV2, MnistDataset, MindDataset, NumpySlicesDataset, \ +from .engine.datasets import TFRecordDataset, ImageFolderDataset, MnistDataset, MindDataset, NumpySlicesDataset, \ GeneratorDataset, ManifestDataset, Cifar10Dataset, Cifar100Dataset, VOCDataset, CocoDataset, CelebADataset, \ TextFileDataset, CLUEDataset, CSVDataset, Schema, Shuffle, zip, RandomDataset, PaddedDataset from .engine.samplers import DistributedSampler, PKSampler, RandomSampler, SequentialSampler, SubsetRandomSampler, \ @@ -28,7 +28,7 @@ from .engine.cache_client import DatasetCache from .engine.serializer_deserializer import serialize, deserialize, show from .engine.graphdata import GraphData -__all__ = ["config", "ImageFolderDatasetV2", "MnistDataset", "PaddedDataset", +__all__ = ["config", "ImageFolderDataset", "MnistDataset", "PaddedDataset", "MindDataset", "GeneratorDataset", "TFRecordDataset", "ManifestDataset", "Cifar10Dataset", "Cifar100Dataset", "CelebADataset", "NumpySlicesDataset", "VOCDataset", "CocoDataset", "TextFileDataset", "CLUEDataset", "CSVDataset", "Schema", "DistributedSampler", "PKSampler", diff --git a/mindspore/dataset/core/py_util_helpers.py b/mindspore/dataset/core/py_util_helpers.py new file mode 100644 index 0000000000..4af5e93eea --- /dev/null +++ b/mindspore/dataset/core/py_util_helpers.py @@ -0,0 +1,31 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +General py_transforms_utils functions. +""" +import numpy as np + + +def is_numpy(img): + """ + Check if the input image is Numpy format. + + Args: + img: Image to be checked. + + Returns: + Bool, True if input is Numpy image. + """ + return isinstance(img, np.ndarray) diff --git a/mindspore/dataset/engine/__init__.py b/mindspore/dataset/engine/__init__.py index 15eb66e54e..015a4a492f 100644 --- a/mindspore/dataset/engine/__init__.py +++ b/mindspore/dataset/engine/__init__.py @@ -28,7 +28,7 @@ from .serializer_deserializer import serialize, deserialize, show, compare from .samplers import * from ..core import config -__all__ = ["config", "zip", "ImageFolderDatasetV2", "MnistDataset", +__all__ = ["config", "zip", "ImageFolderDataset", "MnistDataset", "MindDataset", "GeneratorDataset", "TFRecordDataset", "CLUEDataset", "CSVDataset", "ManifestDataset", "Cifar10Dataset", "Cifar100Dataset", "CelebADataset", "VOCDataset", "CocoDataset", "TextFileDataset", "Schema", "DistributedSampler", diff --git a/mindspore/dataset/engine/datasets.py b/mindspore/dataset/engine/datasets.py index 981974d86a..8524db0254 100644 --- a/mindspore/dataset/engine/datasets.py +++ b/mindspore/dataset/engine/datasets.py @@ -41,7 +41,7 @@ from . import samplers from .iterators import DictIterator, TupleIterator, DummyIterator, SaveOp, Iterator from .validators import check_batch, check_shuffle, check_map, check_filter, check_repeat, check_skip, check_zip, \ check_rename, check_numpyslicesdataset, check_device_send, \ - check_take, check_project, check_imagefolderdatasetv2, check_mnist_cifar_dataset, check_manifestdataset, \ + check_take, check_project, check_imagefolderdataset, check_mnist_cifar_dataset, check_manifestdataset, \ check_tfrecorddataset, check_vocdataset, check_cocodataset, check_celebadataset, check_minddataset, \ check_generatordataset, check_sync_wait, check_zip_dataset, check_add_column, check_textfiledataset, check_concat, \ check_random_dataset, check_split, check_bucket_batch_by_length, check_cluedataset, check_save, check_csvdataset, \ @@ -81,8 +81,8 @@ def zip(datasets): >>> >>> dataset_dir1 = "path/to/imagefolder_directory1" >>> dataset_dir2 = "path/to/imagefolder_directory2" - >>> ds1 = ds.ImageFolderDatasetV2(dataset_dir1, num_parallel_workers=8) - >>> ds2 = ds.ImageFolderDatasetV2(dataset_dir2, num_parallel_workers=8) + >>> ds1 = ds.ImageFolderDataset(dataset_dir1, num_parallel_workers=8) + >>> ds2 = ds.ImageFolderDataset(dataset_dir2, num_parallel_workers=8) >>> >>> # creates a dataset which is the combination of ds1 and ds2 >>> data = ds.zip((ds1, ds2)) @@ -246,7 +246,7 @@ class Dataset: @check_batch def batch(self, batch_size, drop_remainder=False, num_parallel_workers=None, per_batch_map=None, - input_columns=None, pad_info=None): + input_columns=None, output_columns=None, column_order=None, pad_info=None): """ Combine batch_size number of consecutive rows into batches. @@ -272,6 +272,18 @@ class Dataset: The last parameter of the callable should always be a BatchInfo object. input_columns (list[str], optional): List of names of the input columns. The size of the list should match with signature of per_batch_map callable. + output_columns (list[str], optional): [Not currently implmented] List of names assigned to the columns + outputted by the last operation. This parameter is mandatory if len(input_columns) != + len(output_columns). The size of this list must match the number of output + columns of the last operation. (default=None, output columns will have the same + name as the input columns, i.e., the columns will be replaced). + column_order (list[str], optional): [Not currently implmented] list of all the desired columns to + propagate to the child node. This list must be a subset of all the columns in the dataset after + all operations are applied. The order of the columns in each row propagated to the + child node follow the order they appear in this list. The parameter is mandatory + if the len(input_columns) != len(output_columns). (default=None, all columns + will be propagated to the child node, the order of the columns will remain the + same). pad_info (dict, optional): Whether to perform padding on selected columns. pad_info={"col1":([224,224],0)} would pad column with name "col1" to a tensor of size [224,224] and fill the missing with 0. @@ -286,7 +298,7 @@ class Dataset: >>> data = data.batch(100, True) """ return BatchDataset(self, batch_size, drop_remainder, num_parallel_workers, per_batch_map, input_columns, - pad_info) + output_columns, column_order, pad_info) @check_sync_wait def sync_wait(self, condition_name, num_batch=1, callback=None): @@ -367,7 +379,7 @@ class Dataset: >>> # declare a function which returns a Dataset object >>> def flat_map_func(x): >>> data_dir = text.to_str(x[0]) - >>> d = ds.ImageFolderDatasetV2(data_dir) + >>> d = ds.ImageFolderDataset(data_dir) >>> return d >>> # data is a Dataset object >>> data = ds.TextFileDataset(DATA_FILE) @@ -394,7 +406,7 @@ class Dataset: return dataset @check_map - def map(self, input_columns=None, operations=None, output_columns=None, columns_order=None, + def map(self, operations=None, input_columns=None, output_columns=None, column_order=None, num_parallel_workers=None, python_multiprocessing=False, cache=None, callbacks=None): """ Apply each operation in operations to this dataset. @@ -409,23 +421,23 @@ class Dataset: The columns outputted by the very last operation will be assigned names specified by output_columns. - Only the columns specified in columns_order will be propagated to the child node. These - columns will be in the same order as specified in columns_order. + Only the columns specified in column_order will be propagated to the child node. These + columns will be in the same order as specified in column_order. Args: + operations (Union[list[TensorOp], list[functions]]): List of operations to be + applied on the dataset. Operations are applied in the order they appear in this list. input_columns (list[str]): List of the names of the columns that will be passed to the first operation as input. The size of this list must match the number of input columns expected by the first operator. (default=None, the first operation will be passed however many columns that is required, starting from the first column). - operations (Union[list[TensorOp], list[functions]]): List of operations to be - applied on the dataset. Operations are applied in the order they appear in this list. output_columns (list[str], optional): List of names assigned to the columns outputted by the last operation. This parameter is mandatory if len(input_columns) != len(output_columns). The size of this list must match the number of output columns of the last operation. (default=None, output columns will have the same name as the input columns, i.e., the columns will be replaced). - columns_order (list[str], optional): list of all the desired columns to propagate to the + column_order (list[str], optional): list of all the desired columns to propagate to the child node. This list must be a subset of all the columns in the dataset after all operations are applied. The order of the columns in each row propagated to the child node follow the order they appear in this list. The parameter is mandatory @@ -446,7 +458,7 @@ class Dataset: Examples: >>> import mindspore.dataset as ds - >>> import mindspore.dataset.transforms.vision.c_transforms as c_transforms + >>> import mindspore.dataset.vision.c_transforms as c_transforms >>> >>> # data is an instance of Dataset which has 2 columns, "image" and "label". >>> # ds_pyfunc is an instance of Dataset which has 3 columns, "col0", "col1", and "col2". Each column is @@ -468,33 +480,33 @@ class Dataset: >>> input_columns = ["image"] >>> >>> # Applies decode_op on column "image". This column will be replaced by the outputed - >>> # column of decode_op. Since columns_order is not provided, both columns "image" + >>> # column of decode_op. Since column_order is not provided, both columns "image" >>> # and "label" will be propagated to the child node in their original order. - >>> ds_decoded = data.map(input_columns, operations) + >>> ds_decoded = data.map(operations, input_columns) >>> >>> # Rename column "image" to "decoded_image" >>> output_columns = ["decoded_image"] - >>> ds_decoded = data.map(input_columns, operations, output_columns) + >>> ds_decoded = data.map(operations, input_columns, output_columns) >>> >>> # Specify the order of the columns. - >>> columns_order ["label", "image"] - >>> ds_decoded = data.map(input_columns, operations, None, columns_order) + >>> column_order ["label", "image"] + >>> ds_decoded = data.map(operations, input_columns, None, column_order) >>> >>> # Rename column "image" to "decoded_image" and also specify the order of the columns. - >>> columns_order ["label", "decoded_image"] + >>> column_order ["label", "decoded_image"] >>> output_columns = ["decoded_image"] - >>> ds_decoded = data.map(input_columns, operations, output_columns, columns_order) + >>> ds_decoded = data.map(operations, input_columns, output_columns, column_order) >>> >>> # Rename column "image" to "decoded_image" and keep only this column. - >>> columns_order ["decoded_image"] + >>> column_order ["decoded_image"] >>> output_columns = ["decoded_image"] - >>> ds_decoded = data.map(input_columns, operations, output_columns, columns_order) + >>> ds_decoded = data.map(operations, input_columns, output_columns, column_order) >>> >>> # Simple example using pyfunc. Renaming columns and specifying column order >>> # work in the same way as the previous examples. >>> input_columns = ["col0"] >>> operations = [(lambda x: x + 1)] - >>> ds_mapped = ds_pyfunc.map(input_columns, operations) + >>> ds_mapped = ds_pyfunc.map(operations, input_columns) >>> >>> # 2) Map example with more than one operation >>> @@ -509,22 +521,22 @@ class Dataset: >>> # outputted by decode_op is passed as input to random_jitter_op. >>> # random_jitter_op will output one column. Column "image" will be replaced by >>> # the column outputted by random_jitter_op (the very last operation). All other - >>> # columns are unchanged. Since columns_order is not specified, the order of the + >>> # columns are unchanged. Since column_order is not specified, the order of the >>> # columns will remain the same. - >>> ds_mapped = data.map(input_columns, operations) + >>> ds_mapped = data.map(operations, input_columns) >>> >>> # Creates a dataset that is identical to ds_mapped, except the column "image" >>> # that is outputted by random_jitter_op is renamed to "image_transformed". >>> # Specifying column order works in the same way as examples in 1). >>> output_columns = ["image_transformed"] - >>> ds_mapped_and_renamed = data.map(input_columns, operation, output_columns) + >>> ds_mapped_and_renamed = data.map(operation, input_columns, output_columns) >>> >>> # Multiple operations using pyfunc. Renaming columns and specifying column order >>> # work in the same way as examples in 1). >>> input_columns = ["col0"] >>> operations = [(lambda x: x + x), (lambda x: x - 1)] >>> output_columns = ["col0_mapped"] - >>> ds_mapped = ds_pyfunc.map(input_columns, operations, output_columns) + >>> ds_mapped = ds_pyfunc.map(operations, input_columns, output_columns) >>> >>> # 3) Example where number of input columns is not equal to number of output columns >>> @@ -540,20 +552,21 @@ class Dataset: >>> (lambda x: (x % 2, x % 3, x % 5, x % 7))] >>> >>> # Note: because the number of input columns is not the same as the number of - >>> # output columns, the output_columns and columns_order parameter must be + >>> # output columns, the output_columns and column_order parameter must be >>> # specified. Otherwise, this map call will also result in an error. >>> input_columns = ["col2", "col0"] >>> output_columns = ["mod2", "mod3", "mod5", "mod7"] >>> >>> # Propagate all columns to the child node in this order: - >>> columns_order = ["col0", "col2", "mod2", "mod3", "mod5", "mod7", "col1"] - >>> ds_mapped = ds_pyfunc.map(input_columns, operations, output_columns, columns_order) + >>> column_order = ["col0", "col2", "mod2", "mod3", "mod5", "mod7", "col1"] + >>> ds_mapped = ds_pyfunc.map(operations, input_columns, output_columns, column_order) >>> >>> # Propagate some columns to the child node in this order: - >>> columns_order = ["mod7", "mod3", "col1"] - >>> ds_mapped = ds_pyfunc.map(input_columns, operations, output_columns, columns_order) + >>> column_order = ["mod7", "mod3", "col1"] + >>> ds_mapped = ds_pyfunc.map(operations, input_columns, output_columns, column_order) """ - return MapDataset(self, input_columns, operations, output_columns, columns_order, num_parallel_workers, + + return MapDataset(self, operations, input_columns, output_columns, column_order, num_parallel_workers, python_multiprocessing, cache, callbacks) @check_filter @@ -1012,7 +1025,7 @@ class Dataset: def get_distribution(output_dataset): dev_id = 0 - if isinstance(output_dataset, (Cifar10Dataset, Cifar100Dataset, GeneratorDataset, ImageFolderDatasetV2, + if isinstance(output_dataset, (Cifar10Dataset, Cifar100Dataset, GeneratorDataset, ImageFolderDataset, ManifestDataset, MnistDataset, VOCDataset, CocoDataset, CelebADataset, MindDataset)): sampler = output_dataset.sampler @@ -1412,7 +1425,7 @@ class MappableDataset(SourceDataset): >>> >>> dataset_dir = "/path/to/imagefolder_directory" >>> # a SequentialSampler is created by default - >>> data = ds.ImageFolderDatasetV2(dataset_dir) + >>> data = ds.ImageFolderDataset(dataset_dir) >>> >>> # use a DistributedSampler instead of the SequentialSampler >>> new_sampler = ds.DistributedSampler(10, 2) @@ -1501,7 +1514,7 @@ class MappableDataset(SourceDataset): >>> dataset_dir = "/path/to/imagefolder_directory" >>> >>> # many datasets have shuffle on by default, set shuffle to False if split will be called! - >>> data = ds.ImageFolderDatasetV2(dataset_dir, shuffle=False) + >>> data = ds.ImageFolderDataset(dataset_dir, shuffle=False) >>> >>> # sets the seed, and tells split to use this seed when randomizing. This >>> # is needed because we are sharding later @@ -1629,13 +1642,25 @@ class BatchDataset(DatasetOp): last parameter of the callable should always be a BatchInfo object. input_columns (list[str], optional): List of names of the input columns. The size of the list should match with signature of per_batch_map callable. + output_columns (list[str], optional): List of names assigned to the columns outputted by + the last operation. This parameter is mandatory if len(input_columns) != + len(output_columns). The size of this list must match the number of output + columns of the last operation. (default=None, output columns will have the same + name as the input columns, i.e., the columns will be replaced). + column_order (list[str], optional): list of all the desired columns to propagate to the + child node. This list must be a subset of all the columns in the dataset after + all operations are applied. The order of the columns in each row propagated to the + child node follow the order they appear in this list. The parameter is mandatory + if the len(input_columns) != len(output_columns). (default=None, all columns + will be propagated to the child node, the order of the columns will remain the + same). pad_info (dict, optional): Whether to perform padding on selected columns. pad_info={"col1":([224,224],0)} would pad column with name "col1" to a tensor of size [224,224] and fill the missing with 0. """ def __init__(self, input_dataset, batch_size, drop_remainder=False, num_parallel_workers=None, - per_batch_map=None, input_columns=None, pad_info=None): + per_batch_map=None, input_columns=None, output_columns=None, column_order=None, pad_info=None): super().__init__(num_parallel_workers) if BatchDataset._is_ancestor_of_repeat(input_dataset): @@ -1647,6 +1672,8 @@ class BatchDataset(DatasetOp): self.drop_remainder = drop_remainder self.per_batch_map = per_batch_map self.input_columns = input_columns + self.output_columns = output_columns + self.column_order = column_order self.pad_info = pad_info self.children.append(input_dataset) input_dataset.parent.append(self) @@ -1962,16 +1989,16 @@ class MapDataset(DatasetOp): Args: input_dataset (Dataset): Input Dataset to be mapped. + operations (TensorOp): A function mapping a nested structure of tensors + to another nested structure of tensor (default=None). input_columns (list[str]): List of names of the input columns (default=None, the operations will be applied on the first columns in the dataset). The size of the list should match the number of inputs of the first operator. - operations (TensorOp): A function mapping a nested structure of tensors - to another nested structure of tensor (default=None). output_columns (list[str], optional): list of names of the output columns. The size of the list should match the number of outputs of the last operator (default=None, output columns will be the input columns, i.e., the columns will be replaced). - columns_order (list[str], optional): list of all the desired columns of the dataset (default=None). + column_order (list[str], optional): list of all the desired columns of the dataset (default=None). The argument is mandatory if len(input_columns) != len(output_columns). num_parallel_workers (int, optional): Number of workers to process the Dataset in parallel (default=None). @@ -1982,29 +2009,29 @@ class MapDataset(DatasetOp): callbacks: (DSCallback, list[DSCallback], optional): list of Dataset callbacks to be called (Default=None) Raises: - ValueError: If len(input_columns) != len(output_columns) and columns_order is not specified. + ValueError: If len(input_columns) != len(output_columns) and column_order is not specified. """ - def __init__(self, input_dataset, input_columns=None, operations=None, output_columns=None, columns_order=None, + def __init__(self, input_dataset, operations=None, input_columns=None, output_columns=None, column_order=None, num_parallel_workers=None, python_multiprocessing=False, cache=None, callbacks=None): super().__init__(num_parallel_workers) self.children.append(input_dataset) - if input_columns is not None and not isinstance(input_columns, list): - input_columns = [input_columns] - self.input_columns = input_columns if operations is not None and not isinstance(operations, list): operations = [operations] self.operations = operations + if input_columns is not None and not isinstance(input_columns, list): + input_columns = [input_columns] + self.input_columns = input_columns if output_columns is not None and not isinstance(output_columns, list): output_columns = [output_columns] self.output_columns = output_columns self.cache = cache - self.columns_order = columns_order + self.column_order = column_order if self.input_columns and self.output_columns \ and len(self.input_columns) != len(self.output_columns) \ - and self.columns_order is None: - raise ValueError("When (len(input_columns) != len(output_columns)), columns_order must be specified.") + and self.column_order is None: + raise ValueError("When (len(input_columns) != len(output_columns)), column_order must be specified.") input_dataset.parent.append(self) self._input_indexs = input_dataset.input_indexs @@ -2021,7 +2048,7 @@ class MapDataset(DatasetOp): args["input_columns"] = self.input_columns args["operations"] = self.operations args["output_columns"] = self.output_columns - args["columns_order"] = self.columns_order + args["column_order"] = self.column_order args["cache"] = self.cache.cache_client if self.cache is not None else None if self.callbacks is not None: @@ -2048,7 +2075,7 @@ class MapDataset(DatasetOp): new_op.children = copy.deepcopy(self.children, memodict) new_op.input_columns = copy.deepcopy(self.input_columns, memodict) new_op.output_columns = copy.deepcopy(self.output_columns, memodict) - new_op.columns_order = copy.deepcopy(self.columns_order, memodict) + new_op.column_order = copy.deepcopy(self.column_order, memodict) new_op.num_parallel_workers = copy.deepcopy(self.num_parallel_workers, memodict) new_op.parent = copy.deepcopy(self.parent, memodict) new_op.ms_role = copy.deepcopy(self.ms_role, memodict) @@ -2646,7 +2673,7 @@ def _select_sampler(num_samples, input_sampler, shuffle, num_shards, shard_id, n return samplers.SequentialSampler(num_samples=num_samples) -class ImageFolderDatasetV2(MappableDataset): +class ImageFolderDataset(MappableDataset): """ A source dataset that reads images from a tree of directories. @@ -2722,14 +2749,14 @@ class ImageFolderDatasetV2(MappableDataset): >>> # path to imagefolder directory. This directory needs to contain sub-directories which contain the images >>> dataset_dir = "/path/to/imagefolder_directory" >>> # 1) read all samples (image files) in dataset_dir with 8 threads - >>> imagefolder_dataset = ds.ImageFolderDatasetV2(dataset_dir, num_parallel_workers=8) + >>> imagefolder_dataset = ds.ImageFolderDataset(dataset_dir, num_parallel_workers=8) >>> # 2) read all samples (image files) from folder cat and folder dog with label 0 and 1 - >>> imagefolder_dataset = ds.ImageFolderDatasetV2(dataset_dir,class_indexing={"cat":0,"dog":1}) + >>> imagefolder_dataset = ds.ImageFolderDataset(dataset_dir,class_indexing={"cat":0,"dog":1}) >>> # 3) read all samples (image files) in dataset_dir with extensions .JPEG and .png (case sensitive) - >>> imagefolder_dataset = ds.ImageFolderDatasetV2(dataset_dir, extensions=[".JPEG",".png"]) + >>> imagefolder_dataset = ds.ImageFolderDataset(dataset_dir, extensions=[".JPEG",".png"]) """ - @check_imagefolderdatasetv2 + @check_imagefolderdataset def __init__(self, dataset_dir, num_samples=None, num_parallel_workers=None, shuffle=None, sampler=None, extensions=None, class_indexing=None, decode=False, num_shards=None, shard_id=None, cache=None): @@ -3168,6 +3195,7 @@ class SamplerFn: """ Multiprocessing or multithread generator function wrapper master process. """ + def __init__(self, dataset, num_worker, multi_process): self.workers = [] self.num_worker = num_worker diff --git a/mindspore/dataset/engine/iterators.py b/mindspore/dataset/engine/iterators.py index 5981bb3daf..d10cf80999 100644 --- a/mindspore/dataset/engine/iterators.py +++ b/mindspore/dataset/engine/iterators.py @@ -150,7 +150,7 @@ class Iterator: op_type = OpName.SKIP elif isinstance(dataset, de.TakeDataset): op_type = OpName.TAKE - elif isinstance(dataset, de.ImageFolderDatasetV2): + elif isinstance(dataset, de.ImageFolderDataset): op_type = OpName.IMAGEFOLDER elif isinstance(dataset, de.GeneratorDataset): op_type = OpName.GENERATOR diff --git a/mindspore/dataset/engine/samplers.py b/mindspore/dataset/engine/samplers.py index d8fced318f..e3b1e95a9f 100644 --- a/mindspore/dataset/engine/samplers.py +++ b/mindspore/dataset/engine/samplers.py @@ -41,7 +41,7 @@ class Sampler: >>> for i in range(self.dataset_size - 1, -1, -1): >>> yield i >>> - >>> ds = ds.ImageFolderDatasetV2(path, sampler=ReverseSampler()) + >>> ds = ds.ImageFolderDataset(path, sampler=ReverseSampler()) """ def __init__(self, num_samples=None): @@ -232,7 +232,7 @@ class DistributedSampler(BuiltinSampler): >>> >>> # creates a distributed sampler with 10 shards total. This shard is shard 5 >>> sampler = ds.DistributedSampler(10, 5) - >>> data = ds.ImageFolderDatasetV2(dataset_dir, num_parallel_workers=8, sampler=sampler) + >>> data = ds.ImageFolderDataset(dataset_dir, num_parallel_workers=8, sampler=sampler) Raises: ValueError: If num_shards is not positive. @@ -315,7 +315,7 @@ class PKSampler(BuiltinSampler): >>> >>> # creates a PKSampler that will get 3 samples from every class. >>> sampler = ds.PKSampler(3) - >>> data = ds.ImageFolderDatasetV2(dataset_dir, num_parallel_workers=8, sampler=sampler) + >>> data = ds.ImageFolderDataset(dataset_dir, num_parallel_workers=8, sampler=sampler) Raises: ValueError: If num_val is not positive. @@ -387,7 +387,7 @@ class RandomSampler(BuiltinSampler): >>> >>> # creates a RandomSampler >>> sampler = ds.RandomSampler() - >>> data = ds.ImageFolderDatasetV2(dataset_dir, num_parallel_workers=8, sampler=sampler) + >>> data = ds.ImageFolderDataset(dataset_dir, num_parallel_workers=8, sampler=sampler) Raises: ValueError: If replacement is not boolean. @@ -447,7 +447,7 @@ class SequentialSampler(BuiltinSampler): >>> >>> # creates a SequentialSampler >>> sampler = ds.SequentialSampler() - >>> data = ds.ImageFolderDatasetV2(dataset_dir, num_parallel_workers=8, sampler=sampler) + >>> data = ds.ImageFolderDataset(dataset_dir, num_parallel_workers=8, sampler=sampler) """ def __init__(self, start_index=None, num_samples=None): @@ -510,7 +510,7 @@ class SubsetRandomSampler(BuiltinSampler): >>> >>> # creates a SubsetRandomSampler, will sample from the provided indices >>> sampler = ds.SubsetRandomSampler() - >>> data = ds.ImageFolderDatasetV2(dataset_dir, num_parallel_workers=8, sampler=sampler) + >>> data = ds.ImageFolderDataset(dataset_dir, num_parallel_workers=8, sampler=sampler) """ def __init__(self, indices, num_samples=None): @@ -573,7 +573,7 @@ class WeightedRandomSampler(BuiltinSampler): >>> >>> # creates a WeightedRandomSampler that will sample 4 elements without replacement >>> sampler = ds.WeightedRandomSampler(weights, 4) - >>> data = ds.ImageFolderDatasetV2(dataset_dir, num_parallel_workers=8, sampler=sampler) + >>> data = ds.ImageFolderDataset(dataset_dir, num_parallel_workers=8, sampler=sampler) Raises: ValueError: If num_samples is not positive. diff --git a/mindspore/dataset/engine/serializer_deserializer.py b/mindspore/dataset/engine/serializer_deserializer.py index a4fd8d32f7..4a1411a4c6 100644 --- a/mindspore/dataset/engine/serializer_deserializer.py +++ b/mindspore/dataset/engine/serializer_deserializer.py @@ -21,9 +21,10 @@ import sys from mindspore import log as logger from . import datasets as de -from ..transforms.vision.utils import Inter, Border +from ..vision.utils import Inter, Border from ..core import config + def serialize(dataset, json_filepath=None): """ Serialize dataset pipeline into a json file. @@ -44,7 +45,7 @@ def serialize(dataset, json_filepath=None): >>> DATA_DIR = "../../data/testMnistData" >>> data = ds.MnistDataset(DATA_DIR, 100) >>> one_hot_encode = C.OneHot(10) # num_classes is input argument - >>> data = data.map(input_column_names="label", operation=one_hot_encode) + >>> data = data.map(operation=one_hot_encode, input_column_names="label") >>> data = data.batch(batch_size=10, drop_remainder=True) >>> >>> ds.engine.serialize(data, json_filepath="mnist_dataset_pipeline.json") # serialize it to json file @@ -77,7 +78,7 @@ def deserialize(input_dict=None, json_filepath=None): >>> DATA_DIR = "../../data/testMnistData" >>> data = ds.MnistDataset(DATA_DIR, 100) >>> one_hot_encode = C.OneHot(10) # num_classes is input argument - >>> data = data.map(input_column_names="label", operation=one_hot_encode) + >>> data = data.map(operation=one_hot_encode, input_column_names="label") >>> data = data.batch(batch_size=10, drop_remainder=True) >>> >>> # Use case 1: to/from json file @@ -254,7 +255,7 @@ def create_node(node): pyobj = None # Find a matching Dataset class and call the constructor with the corresponding args. # When a new Dataset class is introduced, another if clause and parsing code needs to be added. - if dataset_op == 'ImageFolderDatasetV2': + if dataset_op == 'ImageFolderDataset': sampler = construct_sampler(node.get('sampler')) pyobj = pyclass(node['dataset_dir'], node.get('num_samples'), node.get('num_parallel_workers'), node.get('shuffle'), sampler, node.get('extensions'), @@ -336,8 +337,8 @@ def create_node(node): elif dataset_op == 'MapDataset': tensor_ops = construct_tensor_ops(node.get('operations')) - pyobj = de.Dataset().map(node.get('input_columns'), tensor_ops, node.get('output_columns'), - node.get('columns_order'), node.get('num_parallel_workers')) + pyobj = de.Dataset().map(tensor_ops, node.get('input_columns'), node.get('output_columns'), + node.get('column_order'), node.get('num_parallel_workers')) elif dataset_op == 'ShuffleDataset': pyobj = de.Dataset().shuffle(node.get('buffer_size')) diff --git a/mindspore/dataset/engine/validators.py b/mindspore/dataset/engine/validators.py index 4aae8f2c24..84bfdbc9a1 100644 --- a/mindspore/dataset/engine/validators.py +++ b/mindspore/dataset/engine/validators.py @@ -35,8 +35,8 @@ from . import cache_client from .. import callback -def check_imagefolderdatasetv2(method): - """A wrapper that wraps a parameter checker around the original Dataset(ImageFolderDatasetV2).""" +def check_imagefolderdataset(method): + """A wrapper that wraps a parameter checker around the original Dataset(ImageFolderDataset).""" @wraps(method) def new_method(self, *args, **kwargs): @@ -474,8 +474,8 @@ def check_batch(method): @wraps(method) def new_method(self, *args, **kwargs): - [batch_size, drop_remainder, num_parallel_workers, per_batch_map, - input_columns, pad_info], param_dict = parse_user_args(method, *args, **kwargs) + [batch_size, drop_remainder, num_parallel_workers, per_batch_map, input_columns, output_columns, + column_order, pad_info], param_dict = parse_user_args(method, *args, **kwargs) if not (isinstance(batch_size, int) or (callable(batch_size))): raise TypeError("batch_size should either be an int or a callable.") @@ -510,6 +510,12 @@ def check_batch(method): if len(input_columns) != (len(ins.signature(per_batch_map).parameters) - 1): raise ValueError("the signature of per_batch_map should match with input columns") + if output_columns is not None: + raise ValueError("output_columns is currently not implemented.") + + if column_order is not None: + raise ValueError("column_order is currently not implemented.") + return method(self, *args, **kwargs) return new_method @@ -551,14 +557,14 @@ def check_map(method): @wraps(method) def new_method(self, *args, **kwargs): - [input_columns, _, output_columns, columns_order, num_parallel_workers, python_multiprocessing, cache, + [_, input_columns, output_columns, column_order, num_parallel_workers, python_multiprocessing, cache, callbacks], _ = \ parse_user_args(method, *args, **kwargs) - nreq_param_columns = ['input_columns', 'output_columns', 'columns_order'] + nreq_param_columns = ['input_columns', 'output_columns', 'column_order'] - if columns_order is not None: - type_check(columns_order, (list,), "columns_order") + if column_order is not None: + type_check(column_order, (list,), "column_order") if num_parallel_workers is not None: check_num_parallel_workers(num_parallel_workers) type_check(python_multiprocessing, (bool,), "python_multiprocessing") @@ -571,7 +577,7 @@ def check_map(method): else: type_check(callbacks, (callback.DSCallback,), "callbacks") - for param_name, param in zip(nreq_param_columns, [input_columns, output_columns, columns_order]): + for param_name, param in zip(nreq_param_columns, [input_columns, output_columns, column_order]): if param is not None: check_columns(param, param_name) if callbacks is not None: diff --git a/mindspore/dataset/text/transforms.py b/mindspore/dataset/text/transforms.py index bcd95de46c..a6a74d47a5 100644 --- a/mindspore/dataset/text/transforms.py +++ b/mindspore/dataset/text/transforms.py @@ -162,8 +162,9 @@ class JiebaTokenizer(cde.JiebaTokenizerOp): >>> # If with_offsets=False, then output three columns {["token", dtype=str], ["offsets_start", dtype=uint32], >>> # ["offsets_limit", dtype=uint32]} >>> tokenizer_op = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.MP, with_offsets=True) - >>> data = data.map(input_columns=["text"], output_columns=["token", "offsets_start", "offsets_limit"], - >>> columns_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op) + >>> data = data.map(operations=tokenizer_op, input_columns=["text"], + >>> output_columns=["token", "offsets_start", "offsets_limit"], + >>> column_order=["token", "offsets_start", "offsets_limit"]) """ @check_jieba_init @@ -282,7 +283,7 @@ class UnicodeCharTokenizer(cde.UnicodeCharTokenizerOp): >>> # ["offsets_limit", dtype=uint32]} >>> tokenizer_op = text.UnicodeCharTokenizer(True) >>> data = data.map(input_columns=["text"], output_columns=["token", "offsets_start", "offsets_limit"], - >>> columns_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op) + >>> column_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op) """ @check_with_offsets @@ -313,7 +314,7 @@ class WordpieceTokenizer(cde.WordpieceTokenizerOp): >>> tokenizer_op = text.WordpieceTokenizer(vocab=vocab, unknown_token=['UNK'], >>> max_bytes_per_token=100, with_offsets=True) >>> data = data.map(input_columns=["text"], output_columns=["token", "offsets_start", "offsets_limit"], - >>> columns_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op) + >>> column_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op) """ @check_wordpiece_tokenizer @@ -378,7 +379,7 @@ if platform.system().lower() != 'windows': >>> # ["offsets_limit", dtype=uint32]} >>> tokenizer_op = text.WhitespaceTokenizer(True) >>> data = data.map(input_columns=["text"], output_columns=["token", "offsets_start", "offsets_limit"], - >>> columns_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op) + >>> column_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op) """ @check_with_offsets @@ -404,7 +405,7 @@ if platform.system().lower() != 'windows': >>> # ["offsets_limit", dtype=uint32]} >>> tokenizer_op = text.UnicodeScriptTokenizerOp(keep_whitespace=True, with_offsets=True) >>> data = data.map(input_columns=["text"], output_columns=["token", "offsets_start", "offsets_limit"], - >>> columns_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op) + >>> column_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op) """ @check_unicode_script_tokenizer @@ -497,7 +498,7 @@ if platform.system().lower() != 'windows': >>> # ["offsets_limit", dtype=uint32]} >>> tokenizer_op = text.RegexTokenizer(delim_pattern, keep_delim_pattern, with_offsets=True) >>> data = data.map(input_columns=["text"], output_columns=["token", "offsets_start", "offsets_limit"], - >>> columns_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op) + >>> column_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op) """ @check_regex_tokenizer @@ -540,7 +541,7 @@ if platform.system().lower() != 'windows': >>> preserve_unused_token=True, >>> with_offsets=True) >>> data = data.map(input_columns=["text"], output_columns=["token", "offsets_start", "offsets_limit"], - >>> columns_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op) + >>> column_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op) """ @check_basic_tokenizer @@ -593,7 +594,7 @@ if platform.system().lower() != 'windows': >>> normalization_form=NormalizeForm.NONE, preserve_unused_token=True, >>> with_offsets=True) >>> data = data.map(input_columns=["text"], output_columns=["token", "offsets_start", "offsets_limit"], - >>> columns_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op) + >>> column_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op) """ @check_bert_tokenizer diff --git a/mindspore/dataset/transforms/__init__.py b/mindspore/dataset/transforms/__init__.py index 73d74a4b1e..c4da2630c7 100644 --- a/mindspore/dataset/transforms/__init__.py +++ b/mindspore/dataset/transforms/__init__.py @@ -16,6 +16,6 @@ This module is to support common augmentations. C_transforms is a high performan image augmentation module which is developed with C++ OpenCV. Py_transforms provide more kinds of image augmentations which is developed with Python PIL. """ -from . import vision +from .. import vision from . import c_transforms from . import py_transforms diff --git a/mindspore/dataset/transforms/c_transforms.py b/mindspore/dataset/transforms/c_transforms.py index 21b8d2d9c0..96f8051d12 100644 --- a/mindspore/dataset/transforms/c_transforms.py +++ b/mindspore/dataset/transforms/c_transforms.py @@ -229,8 +229,8 @@ class Duplicate(cde.DuplicateOp): >>> # +---------+ >>> # | [1,2,3] | >>> # +---------+ - >>> data = data.map(input_columns=["x"], operations=Duplicate(), - >>> output_columns=["x", "y"], columns_order=["x", "y"]) + >>> data = data.map(operations=Duplicate(), input_columns=["x"], + >>> output_columns=["x", "y"], column_order=["x", "y"]) >>> # Data after >>> # | x | y | >>> # +---------+---------+ diff --git a/mindspore/dataset/transforms/py_transforms.py b/mindspore/dataset/transforms/py_transforms.py index 43b2eafee3..475585f525 100644 --- a/mindspore/dataset/transforms/py_transforms.py +++ b/mindspore/dataset/transforms/py_transforms.py @@ -17,9 +17,8 @@ This module py_transforms is implemented basing on Python. It provides common operations including OneHotOp. """ - -from .validators import check_one_hot_op -from .vision import py_transforms_util as util +from .validators import check_one_hot_op, check_compose_list +from . import py_transforms_util as util class OneHotOp: @@ -48,3 +47,48 @@ class OneHotOp: label (numpy.ndarray), label after being Smoothed. """ return util.one_hot_encoding(label, self.num_classes, self.smoothing_rate) + + +class Compose: + """ + Compose a list of transforms. + + .. Note:: + Compose takes a list of transformations either provided in py_transforms or from user-defined implementation; + each can be an initialized transformation class or a lambda function, as long as the output from the last + transformation is a single tensor of type numpy.ndarray. See below for an example of how to use Compose + with py_transforms classes and check out FiveCrop or TenCrop for the use of them in conjunction with lambda + functions. + + Args: + transforms (list): List of transformations to be applied. + + Examples: + >>> import mindspore.dataset as ds + >>> import mindspore.dataset.vision.py_transforms as py_transforms + >>> from mindspore.dataset.transforms.py_transforms import Compose + >>> dataset_dir = "path/to/imagefolder_directory" + >>> # create a dataset that reads all files in dataset_dir with 8 threads + >>> dataset = ds.ImageFolderDataset(dataset_dir, num_parallel_workers=8) + >>> # create a list of transformations to be applied to the image data + >>> transform = Compose([py_transforms.Decode(), + >>> py_transforms.RandomHorizontalFlip(0.5), + >>> py_transforms.ToTensor(), + >>> py_transforms.Normalize((0.491, 0.482, 0.447), (0.247, 0.243, 0.262)), + >>> py_transforms.RandomErasing()]) + >>> # apply the transform to the dataset through dataset.map() + >>> dataset = dataset.map(operations=transform, input_columns="image") + """ + + @check_compose_list + def __init__(self, transforms): + self.transforms = transforms + + def __call__(self, img): + """ + Call method. + + Returns: + lambda function, Lambda function that takes in an img to apply transformations on. + """ + return util.compose(img, self.transforms) diff --git a/mindspore/dataset/transforms/py_transforms_util.py b/mindspore/dataset/transforms/py_transforms_util.py new file mode 100644 index 0000000000..a4b85846eb --- /dev/null +++ b/mindspore/dataset/transforms/py_transforms_util.py @@ -0,0 +1,65 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +Built-in py_transforms_utils functions. +""" +import numpy as np +from ..core.py_util_helpers import is_numpy + + +def compose(img, transforms): + """ + Compose a list of transforms and apply on the image. + + Args: + img (numpy.ndarray): An image in Numpy ndarray. + transforms (list): A list of transform Class objects to be composed. + + Returns: + img (numpy.ndarray), An augmented image in Numpy ndarray. + """ + if is_numpy(img): + for transform in transforms: + img = transform(img) + if is_numpy(img): + return img + raise TypeError('img should be Numpy ndarray. Got {}. Append ToTensor() to transforms'.format(type(img))) + raise TypeError('img should be Numpy ndarray. Got {}.'.format(type(img))) + + +def one_hot_encoding(label, num_classes, epsilon): + """ + Apply label smoothing transformation to the input label, and make label be more smoothing and continuous. + + Args: + label (numpy.ndarray): label to be applied label smoothing. + num_classes (int): Num class of object in dataset, value should over 0. + epsilon (float): The adjustable Hyper parameter. Default is 0.0. + + Returns: + img (numpy.ndarray), label after being one hot encoded and done label smoothed. + """ + if label > num_classes: + raise ValueError('the num_classes is smaller than the category number.') + + num_elements = label.size + one_hot_label = np.zeros((num_elements, num_classes), dtype=int) + + if isinstance(label, list) is False: + label = [label] + for index in range(num_elements): + one_hot_label[index, label[index]] = 1 + + return (1 - epsilon) * one_hot_label + epsilon / num_classes diff --git a/mindspore/dataset/transforms/validators.py b/mindspore/dataset/transforms/validators.py index f44fd918ee..0145d59018 100644 --- a/mindspore/dataset/transforms/validators.py +++ b/mindspore/dataset/transforms/validators.py @@ -200,3 +200,19 @@ def check_random_transform_ops(method): return method(self, *args, **kwargs) return new_method + + +def check_compose_list(method): + """Wrapper method to check the transform list of Compose.""" + + @wraps(method) + def new_method(self, *args, **kwargs): + [transforms], _ = parse_user_args(method, *args, **kwargs) + + type_check(transforms, (list,), transforms) + if not transforms: + raise ValueError("transforms list is empty.") + + return method(self, *args, **kwargs) + + return new_method diff --git a/mindspore/dataset/transforms/vision/__init__.py b/mindspore/dataset/vision/__init__.py similarity index 100% rename from mindspore/dataset/transforms/vision/__init__.py rename to mindspore/dataset/vision/__init__.py diff --git a/mindspore/dataset/transforms/vision/c_transforms.py b/mindspore/dataset/vision/c_transforms.py similarity index 99% rename from mindspore/dataset/transforms/vision/c_transforms.py rename to mindspore/dataset/vision/c_transforms.py index b2d5c06c59..9dd186a04a 100644 --- a/mindspore/dataset/transforms/vision/c_transforms.py +++ b/mindspore/dataset/vision/c_transforms.py @@ -25,11 +25,12 @@ to improve their training models. Examples: >>> import mindspore.dataset as ds >>> import mindspore.dataset.transforms.c_transforms as c_transforms - >>> import mindspore.dataset.transforms.vision.c_transforms as c_vision + >>> import mindspore.dataset.vision.c_transforms as c_vision >>> from mindspore.dataset.transforms.vision.utils import Border, ImageBatchFormat, Inter + >>> dataset_dir = "path/to/imagefolder_directory" >>> # create a dataset that reads all files in dataset_dir with 8 threads - >>> data1 = ds.ImageFolderDatasetV2(dataset_dir, num_parallel_workers=8) + >>> data1 = ds.ImageFolderDataset(dataset_dir, num_parallel_workers=8) >>> # create a list of transformations to be applied to the image data >>> transforms_list = [c_vision.Decode(), >>> c_vision.Resize((256, 256)), @@ -1095,7 +1096,7 @@ class UniformAugment(cde.UniformAugOp): num_ops (int, optional): Number of operations to be selected and applied (default=2). Examples: - >>> import mindspore.dataset.transforms.vision.py_transforms as py_vision + >>> import mindspore.dataset.vision.py_transforms as py_vision >>> transforms_list = [c_vision.RandomHorizontalFlip(), >>> c_vision.RandomVerticalFlip(), >>> c_vision.RandomColorAdjust(), diff --git a/mindspore/dataset/transforms/vision/py_transforms.py b/mindspore/dataset/vision/py_transforms.py similarity index 80% rename from mindspore/dataset/transforms/vision/py_transforms.py rename to mindspore/dataset/vision/py_transforms.py index 86fa9556ff..4973c90adf 100644 --- a/mindspore/dataset/transforms/vision/py_transforms.py +++ b/mindspore/dataset/vision/py_transforms.py @@ -33,7 +33,7 @@ from .validators import check_prob, check_crop, check_resize_interpolation, chec check_normalize_py, check_random_crop, check_random_color_adjust, check_random_rotation, \ check_transforms_list, check_random_apply, check_ten_crop, check_num_channels, check_pad, \ check_random_perspective, check_random_erasing, check_cutout, check_linear_transform, check_random_affine, \ - check_mix_up, check_positive_degrees, check_uniform_augment_py, check_compose_list, check_auto_contrast + check_mix_up, check_positive_degrees, check_uniform_augment_py, check_auto_contrast from .utils import Inter, Border DE_PY_INTER_MODE = {Inter.NEAREST: Image.NEAREST, @@ -46,50 +46,6 @@ DE_PY_BORDER_TYPE = {Border.CONSTANT: 'constant', Border.SYMMETRIC: 'symmetric'} -class ComposeOp: - """ - Compose a list of transforms. - - .. Note:: - ComposeOp takes a list of transformations either provided in py_transforms or from user-defined implementation; - each can be an initialized transformation class or a lambda function, as long as the output from the last - transformation is a single tensor of type numpy.ndarray. See below for an example of how to use ComposeOp - with py_transforms classes and check out FiveCrop or TenCrop for the use of them in conjunction with lambda - functions. - - Args: - transforms (list): List of transformations to be applied. - - Examples: - >>> import mindspore.dataset as ds - >>> import mindspore.dataset.transforms.vision.py_transforms as py_transforms - >>> dataset_dir = "path/to/imagefolder_directory" - >>> # create a dataset that reads all files in dataset_dir with 8 threads - >>> dataset = ds.ImageFolderDatasetV2(dataset_dir, num_parallel_workers=8) - >>> # create a list of transformations to be applied to the image data - >>> transform = py_transforms.ComposeOp([py_transforms.Decode(), - >>> py_transforms.RandomHorizontalFlip(0.5), - >>> py_transforms.ToTensor(), - >>> py_transforms.Normalize((0.491, 0.482, 0.447), (0.247, 0.243, 0.262)), - >>> py_transforms.RandomErasing()]) - >>> # apply the transform to the dataset through dataset.map() - >>> dataset = dataset.map(input_columns="image", operations=transform()) - """ - - @check_compose_list - def __init__(self, transforms): - self.transforms = transforms - - def __call__(self): - """ - Call method. - - Returns: - lambda function, Lambda function that takes in an image to apply transformations on. - """ - return lambda img: util.compose(img, self.transforms) - - class ToTensor: """ Convert the input NumPy image array or PIL image of shape (H,W,C) to a NumPy ndarray of shape (C,H,W). @@ -103,9 +59,11 @@ class ToTensor: output_type (Numpy datatype, optional): The datatype of the NumPy output (default=np.float32). Examples: - >>> py_transforms.ComposeOp([py_transforms.Decode(), - >>> py_transforms.RandomHorizontalFlip(0.5), - >>> py_transforms.ToTensor()]) + >>> import mindspore.dataset.vision.py_transforms as py_transforms + >>> from mindspore.dataset.transforms.py_transforms import Compose + >>> + >>> Compose([py_transforms.Decode(), py_transforms.RandomHorizontalFlip(0.5), + >>> py_transforms.ToTensor()]) """ def __init__(self, output_type=np.float32): @@ -132,11 +90,13 @@ class ToType: output_type (Numpy datatype): The datatype of the NumPy output, e.g. numpy.float32. Examples: + >>> import mindspore.dataset.vision.py_transforms as py_transforms + >>> from mindspore.dataset.transforms.py_transforms import Compose >>> import numpy as np - >>> py_transforms.ComposeOp([py_transforms.Decode(), - >>> py_transforms.RandomHorizontalFlip(0.5), - >>> py_transforms.ToTensor(), - >>> py_transforms.ToType(np.float32)]) + >>> + >>> Compose([py_transforms.Decode(), py_transforms.RandomHorizontalFlip(0.5), + >>> py_transforms.ToTensor(), + >>> py_transforms.ToType(np.float32)]) """ def __init__(self, output_type): @@ -179,9 +139,11 @@ class ToPIL: Examples: >>> # data is already decoded, but not in PIL image format - >>> py_transforms.ComposeOp([py_transforms.ToPIL(), - >>> py_transforms.RandomHorizontalFlip(0.5), - >>> py_transforms.ToTensor()]) + >>> import mindspore.dataset.vision.py_transforms as py_transforms + >>> from mindspore.dataset.transforms.py_transforms import Compose + >>> + >>> Compose([py_transforms.ToPIL(), py_transforms.RandomHorizontalFlip(0.5), + >>> py_transforms.ToTensor()]) """ def __call__(self, img): @@ -202,7 +164,10 @@ class Decode: Decode the input image to PIL image format in RGB mode. Examples: - >>> py_transforms.ComposeOp([py_transforms.Decode(), + >>> import mindspore.dataset.vision.py_transforms as py_transforms + >>> from mindspore.dataset.transforms.py_transforms import Compose + >>> + >>> Compose([py_transforms.Decode(), >>> py_transforms.RandomHorizontalFlip(0.5), >>> py_transforms.ToTensor()]) """ @@ -233,10 +198,13 @@ class Normalize: The standard deviation values must be in range (0.0, 1.0]. Examples: - >>> py_transforms.ComposeOp([py_transforms.Decode(), - >>> py_transforms.RandomHorizontalFlip(0.5), - >>> py_transforms.ToTensor(), - >>> py_transforms.Normalize((0.491, 0.482, 0.447), (0.247, 0.243, 0.262))]) + >>> import mindspore.dataset.vision.py_transforms as py_transforms + >>> from mindspore.dataset.transforms.py_transforms import Compose + >>> + >>> Compose([py_transforms.Decode(), + >>> py_transforms.RandomHorizontalFlip(0.5), + >>> py_transforms.ToTensor(), + >>> py_transforms.Normalize((0.491, 0.482, 0.447), (0.247, 0.243, 0.262))]) """ @check_normalize_py @@ -291,9 +259,12 @@ class RandomCrop: value of edge. Examples: - >>> py_transforms.ComposeOp([py_transforms.Decode(), - >>> py_transforms.RandomCrop(224), - >>> py_transforms.ToTensor()]) + >>> import mindspore.dataset.vision.py_transforms as py_transforms + >>> from mindspore.dataset.transforms.py_transforms import Compose + >>> + >>> Compose([py_transforms.Decode(), + >>> py_transforms.RandomCrop(224), + >>> py_transforms.ToTensor()]) """ @check_random_crop @@ -330,9 +301,12 @@ class RandomHorizontalFlip: prob (float, optional): Probability of the image being flipped (default=0.5). Examples: - >>> py_transforms.ComposeOp([py_transforms.Decode(), - >>> py_transforms.RandomHorizontalFlip(0.5), - >>> py_transforms.ToTensor()]) + >>> import mindspore.dataset.vision.py_transforms as py_transforms + >>> from mindspore.dataset.transforms.py_transforms import Compose + >>> + >>> Compose([py_transforms.Decode(), + >>> py_transforms.RandomHorizontalFlip(0.5), + >>> py_transforms.ToTensor()]) """ @check_prob @@ -360,9 +334,12 @@ class RandomVerticalFlip: prob (float, optional): Probability of the image being flipped (default=0.5). Examples: - >>> py_transforms.ComposeOp([py_transforms.Decode(), - >>> py_transforms.RandomVerticalFlip(0.5), - >>> py_transforms.ToTensor()]) + >>> import mindspore.dataset.vision.py_transforms as py_transforms + >>> from mindspore.dataset.transforms.py_transforms import Compose + >>> + >>> Compose([py_transforms.Decode(), + >>> py_transforms.RandomVerticalFlip(0.5), + >>> py_transforms.ToTensor()]) """ @check_prob @@ -401,9 +378,12 @@ class Resize: - Inter.BICUBIC, means interpolation method is bicubic interpolation. Examples: - >>> py_transforms.ComposeOp([py_transforms.Decode(), - >>> py_transforms.Resize(256), - >>> py_transforms.ToTensor()]) + >>> import mindspore.dataset.vision.py_transforms as py_transforms + >>> from mindspore.dataset.transforms.py_transforms import Compose + >>> + >>> Compose([py_transforms.Decode(), + >>> py_transforms.Resize(256), + >>> py_transforms.ToTensor()]) """ @check_resize_interpolation @@ -448,9 +428,12 @@ class RandomResizedCrop: crop area (default=10). If exceeded, fall back to use center crop instead. Examples: - >>> py_transforms.ComposeOp([py_transforms.Decode(), - >>> py_transforms.RandomResizedCrop(224), - >>> py_transforms.ToTensor()]) + >>> import mindspore.dataset.vision.py_transforms as py_transforms + >>> from mindspore.dataset.transforms.py_transforms import Compose + >>> + >>> Compose([py_transforms.Decode(), + >>> py_transforms.RandomResizedCrop(224), + >>> py_transforms.ToTensor()]) """ @check_random_resize_crop @@ -486,9 +469,12 @@ class CenterCrop: If size is a sequence of length 2, it should be (height, width). Examples: - >>> py_transforms.ComposeOp([py_transforms.Decode(), - >>> py_transforms.CenterCrop(64), - >>> py_transforms.ToTensor()]) + >>> import mindspore.dataset.vision.py_transforms as py_transforms + >>> from mindspore.dataset.transforms.py_transforms import Compose + >>> + >>> Compose([py_transforms.Decode(), + >>> py_transforms.CenterCrop(64), + >>> py_transforms.ToTensor()]) """ @check_crop @@ -527,9 +513,12 @@ class RandomColorAdjust: If it is a sequence, it should be [min, max] where -0.5 <= min <= max <= 0.5. Examples: - >>> py_transforms.ComposeOp([py_transforms.Decode(), - >>> py_transforms.RandomColorAdjust(0.4, 0.4, 0.4, 0.1), - >>> py_transforms.ToTensor()]) + >>> import mindspore.dataset.vision.py_transforms as py_transforms + >>> from mindspore.dataset.transforms.py_transforms import Compose + >>> + >>> Compose([py_transforms.Decode(), + >>> py_transforms.RandomColorAdjust(0.4, 0.4, 0.4, 0.1), + >>> py_transforms.ToTensor()]) """ @check_random_color_adjust @@ -585,9 +574,12 @@ class RandomRotation: If it is an int, it is used for all RGB channels. Default is 0. Examples: - >>> py_transforms.ComposeOp([py_transforms.Decode(), - >>> py_transforms.RandomRotation(30), - >>> py_transforms.ToTensor()]) + >>> import mindspore.dataset.vision.py_transforms as py_transforms + >>> from mindspore.dataset.transforms.py_transforms import Compose + >>> + >>> Compose([py_transforms.Decode(), + >>> py_transforms.RandomRotation(30), + >>> py_transforms.ToTensor()]) """ @check_random_rotation @@ -619,10 +611,12 @@ class RandomOrder: transforms (list): List of the transformations to be applied. Examples: - >>> transforms_list = [py_transforms.CenterCrop(64), py_transforms.RandomRotation(30)] - >>> py_transforms.ComposeOp([py_transforms.Decode(), - >>> py_transforms.RandomOrder(transforms_list), - >>> py_transforms.ToTensor()]) + >>> import mindspore.dataset.vision.py_transforms as py_transforms + >>> from mindspore.dataset.transforms.py_transforms import Compose + >>> + >>> Compose([py_transforms.Decode(), + >>> py_transforms.RandomOrder(transforms_list), + >>> py_transforms.ToTensor()]) """ @check_transforms_list @@ -651,10 +645,12 @@ class RandomApply: prob (float, optional): The probability to apply the transformation list (default=0.5). Examples: - >>> transforms_list = [py_transforms.CenterCrop(64), py_transforms.RandomRotation(30)] - >>> py_transforms.ComposeOp([py_transforms.Decode(), - >>> py_transforms.RandomApply(transforms_list, prob=0.6), - >>> py_transforms.ToTensor()]) + >>> import mindspore.dataset.vision.py_transforms as py_transforms + >>> from mindspore.dataset.transforms.py_transforms import Compose + >>> + >>> Compose([py_transforms.Decode(), + >>> py_transforms.RandomApply(transforms_list, prob=0.6), + >>> py_transforms.ToTensor()]) """ @check_random_apply @@ -683,10 +679,12 @@ class RandomChoice: transforms (list): List of transformations to be chosen from to apply. Examples: - >>> transforms_list = [py_transforms.CenterCrop(64), py_transforms.RandomRotation(30)] - >>> py_transforms.ComposeOp([py_transforms.Decode(), - >>> py_transforms.RandomChoice(transforms_list), - >>> py_transforms.ToTensor()]) + >>> import mindspore.dataset.vision.py_transforms as py_transforms + >>> from mindspore.dataset.transforms.py_transforms import Compose + >>> + >>> Compose([py_transforms.Decode(), + >>> py_transforms.RandomChoice(transforms_list), + >>> py_transforms.ToTensor()]) """ @check_transforms_list @@ -716,10 +714,13 @@ class FiveCrop: If size is a sequence of length 2, it should be (height, width). Examples: - >>> py_transforms.ComposeOp([py_transforms.Decode(), - >>> py_transforms.FiveCrop(size), - >>> # 4D stack of 5 images - >>> lambda images: numpy.stack([py_transforms.ToTensor()(image) for image in images])]) + >>> import mindspore.dataset.vision.py_transforms as py_transforms + >>> from mindspore.dataset.transforms.py_transforms import Compose + >>> + >>> Compose([py_transforms.Decode(), + >>> py_transforms.FiveCrop(size), + >>> # 4D stack of 5 images + >>> lambda images: numpy.stack([py_transforms.ToTensor()(image) for image in images])]) """ @check_crop @@ -752,10 +753,13 @@ class TenCrop: if set to True (default=False). Examples: - >>> py_transforms.ComposeOp([py_transforms.Decode(), - >>> py_transforms.TenCrop(size), - >>> # 4D stack of 10 images - >>> lambda images: numpy.stack([py_transforms.ToTensor()(image) for image in images])]) + >>> import mindspore.dataset.vision.py_transforms as py_transforms + >>> from mindspore.dataset.transforms.py_transforms import Compose + >>> + >>> Compose([py_transforms.Decode(), + >>> py_transforms.TenCrop(size), + >>> # 4D stack of 10 images + >>> lambda images: numpy.stack([py_transforms.ToTensor()(image) for image in images])]) """ @check_ten_crop @@ -789,9 +793,12 @@ class Grayscale: Default is 1. If set to 3, the returned image has 3 identical RGB channels. Examples: - >>> py_transforms.ComposeOp([py_transforms.Decode(), - >>> py_transforms.Grayscale(3), - >>> py_transforms.ToTensor()]) + >>> import mindspore.dataset.vision.py_transforms as py_transforms + >>> from mindspore.dataset.transforms.py_transforms import Compose + >>> + >>> Compose([py_transforms.Decode(), + >>> py_transforms.Grayscale(3), + >>> py_transforms.ToTensor()]) """ @check_num_channels @@ -819,9 +826,12 @@ class RandomGrayscale: prob (float, optional): Probability of the image being converted to grayscale (default=0.1). Examples: - >>> py_transforms.ComposeOp([py_transforms.Decode(), - >>> py_transforms.RandomGrayscale(0.3), - >>> py_transforms.ToTensor()]) + >>> import mindspore.dataset.vision.py_transforms as py_transforms + >>> from mindspore.dataset.transforms.py_transforms import Compose + >>> + >>> Compose([py_transforms.Decode(), + >>> py_transforms.RandomGrayscale(0.3), + >>> py_transforms.ToTensor()]) """ @check_prob @@ -878,10 +888,13 @@ class Pad: value of edge. Examples: - >>> py_transforms.ComposeOp([py_transforms.Decode(), - >>> # adds 10 pixels (default black) to each side of the border of the image - >>> py_transforms.Pad(padding=10), - >>> py_transforms.ToTensor()]) + >>> import mindspore.dataset.vision.py_transforms as py_transforms + >>> from mindspore.dataset.transforms.py_transforms import Compose + >>> + >>> Compose([py_transforms.Decode(), + >>> # adds 10 pixels (default black) to each side of the border of the image + >>> py_transforms.Pad(padding=10), + >>> py_transforms.ToTensor()]) """ @check_pad @@ -922,9 +935,12 @@ class RandomPerspective: - Inter.BICUBIC, means interpolation method is bicubic interpolation. Examples: - >>> py_transforms.ComposeOp([py_transforms.Decode(), - >>> py_transforms.RandomPerspective(prob=0.1), - >>> py_transforms.ToTensor()]) + >>> import mindspore.dataset.vision.py_transforms as py_transforms + >>> from mindspore.dataset.transforms.py_transforms import Compose + >>> + >>> Compose([py_transforms.Decode(), + >>> py_transforms.RandomPerspective(prob=0.1), + >>> py_transforms.ToTensor()]) """ @check_random_perspective @@ -972,9 +988,12 @@ class RandomErasing: erase_area (default=10). If exceeded, return the original image. Examples: - >>> py_transforms.ComposeOp([py_transforms.Decode(), - >>> py_transforms.ToTensor(), - >>> py_transforms.RandomErasing(value='random')]) + >>> import mindspore.dataset.vision.py_transforms as py_transforms + >>> from mindspore.dataset.transforms.py_transforms import Compose + >>> + >>> Compose([py_transforms.Decode(), + >>> py_transforms.ToTensor(), + >>> py_transforms.RandomErasing(value='random')]) """ @check_random_erasing @@ -1016,9 +1035,12 @@ class Cutout: num_patches (int, optional): Number of patches to be cut out of an image (default=1). Examples: - >>> py_transforms.ComposeOp([py_transforms.Decode(), - >>> py_transforms.ToTensor(), - >>> py_transforms.Cutout(80)]) + >>> import mindspore.dataset.vision.py_transforms as py_transforms + >>> from mindspore.dataset.transforms.py_transforms import Compose + >>> + >>> Compose([py_transforms.Decode(), + >>> py_transforms.ToTensor(), + >>> py_transforms.Cutout(80)]) """ @check_cutout @@ -1043,7 +1065,8 @@ class Cutout: bounded = False for _ in range(self.num_patches): - i, j, erase_h, erase_w, erase_value = util.get_erase_params(np_img, (scale, scale), (1, 1), 0, bounded, 1) + i, j, erase_h, erase_w, erase_value = util.get_erase_params(np_img, (scale, scale), (1, 1), 0, bounded, + 1) np_img = util.erase(np_img, i, j, erase_h, erase_w, erase_value) return np_img @@ -1061,10 +1084,13 @@ class LinearTransformation: mean_vector (numpy.ndarray): a NumPy ndarray of shape (D,) where D = C x H x W. Examples: - >>> py_transforms.ComposeOp([py_transforms.Decode(), - >>> py_transforms.Resize(256), - >>> py_transforms.ToTensor(), - >>> py_transforms.LinearTransformation(transformation_matrix, mean_vector)]) + >>> import mindspore.dataset.vision.py_transforms as py_transforms + >>> from mindspore.dataset.transforms.py_transforms import Compose + >>> + >>> Compose([py_transforms.Decode(), + >>> py_transforms.Resize(256), + >>> py_transforms.ToTensor(), + >>> py_transforms.LinearTransformation(transformation_matrix, mean_vector)]) """ @check_linear_transform @@ -1133,9 +1159,12 @@ class RandomAffine: TypeError: If fill_value is not a single integer or a 3-tuple. Examples: - >>> py_transforms.ComposeOp([py_transforms.Decode(), - >>> py_transforms.RandomAffine(degrees=15, translate=(0.1, 0.1), scale=(0.9, 1.1)), - >>> py_transforms.ToTensor()]) + >>> import mindspore.dataset.vision.py_transforms as py_transforms + >>> from mindspore.dataset.transforms.py_transforms import Compose + >>> + >>> Compose([py_transforms.Decode(), + >>> py_transforms.RandomAffine(degrees=15, translate=(0.1, 0.1), scale=(0.9, 1.1)), + >>> py_transforms.ToTensor()]) """ @check_random_affine @@ -1278,9 +1307,12 @@ class RandomColor: It should be in (min, max) format (default=(0.1,1.9)). Examples: - >>> py_transforms.ComposeOp([py_transforms.Decode(), - >>> py_transforms.RandomColor((0.5,1.5)), - >>> py_transforms.ToTensor()]) + >>> import mindspore.dataset.vision.py_transforms as py_transforms + >>> from mindspore.dataset.transforms.py_transforms import Compose + >>> + >>> Compose([py_transforms.Decode(), + >>> py_transforms.RandomColor((0.5,1.5)), + >>> py_transforms.ToTensor()]) """ @check_positive_degrees @@ -1310,9 +1342,12 @@ class RandomSharpness: It should be in (min, max) format (default=(0.1,1.9)). Examples: - >>> py_transforms.ComposeOp([py_transforms.Decode(), - >>> py_transforms.RandomSharpness((0.5,1.5)), - >>> py_transforms.ToTensor()]) + >>> import mindspore.dataset.vision.py_transforms as py_transforms + >>> from mindspore.dataset.transforms.py_transforms import Compose + >>> + >>> Compose([py_transforms.Decode(), + >>> py_transforms.RandomSharpness((0.5,1.5)), + >>> py_transforms.ToTensor()]) """ @@ -1343,9 +1378,12 @@ class AutoContrast: ignore (Union[int, sequence], optional): Pixel values to ignore (default=None). Examples: - >>> py_transforms.ComposeOp([py_transforms.Decode(), - >>> py_transforms.AutoContrast(), - >>> py_transforms.ToTensor()]) + >>> import mindspore.dataset.vision.py_transforms as py_transforms + >>> from mindspore.dataset.transforms.py_transforms import Compose + >>> + >>> Compose([py_transforms.Decode(), + >>> py_transforms.AutoContrast(), + >>> py_transforms.ToTensor()]) """ @@ -1373,9 +1411,12 @@ class Invert: Invert colors of input PIL image. Examples: - >>> py_transforms.ComposeOp([py_transforms.Decode(), - >>> py_transforms.Invert(), - >>> py_transforms.ToTensor()]) + >>> import mindspore.dataset.vision.py_transforms as py_transforms + >>> from mindspore.dataset.transforms.py_transforms import Compose + >>> + >>> Compose([py_transforms.Decode(), + >>> py_transforms.Invert(), + >>> py_transforms.ToTensor()]) """ @@ -1398,9 +1439,12 @@ class Equalize: Equalize the histogram of input PIL image. Examples: - >>> py_transforms.ComposeOp([py_transforms.Decode(), - >>> py_transforms.Equalize(), - >>> py_transforms.ToTensor()]) + >>> import mindspore.dataset.vision.py_transforms as py_transforms + >>> from mindspore.dataset.transforms.py_transforms import Compose + >>> + >>> Compose([py_transforms.Decode(), + >>> py_transforms.Equalize(), + >>> py_transforms.ToTensor()]) """ @@ -1430,13 +1474,16 @@ class UniformAugment: num_ops (int, optional): number of transforms to sequentially apply (default=2). Examples: + >>> import mindspore.dataset.vision.py_transforms as py_transforms + >>> from mindspore.dataset.transforms.py_transforms import Compose + >>> >>> transforms_list = [py_transforms.CenterCrop(64), >>> py_transforms.RandomColor(), >>> py_transforms.RandomSharpness(), >>> py_transforms.RandomRotation(30)] - >>> py_transforms.ComposeOp([py_transforms.Decode(), - >>> py_transforms.UniformAugment(transforms_list), - >>> py_transforms.ToTensor()]) + >>> Compose([py_transforms.Decode(), + >>> py_transforms.UniformAugment(transforms_list), + >>> py_transforms.ToTensor()]) """ @check_uniform_augment_py diff --git a/mindspore/dataset/transforms/vision/py_transforms_util.py b/mindspore/dataset/vision/py_transforms_util.py similarity index 96% rename from mindspore/dataset/transforms/vision/py_transforms_util.py rename to mindspore/dataset/vision/py_transforms_util.py index 1c6c5b1e6b..d751bd11d9 100644 --- a/mindspore/dataset/transforms/vision/py_transforms_util.py +++ b/mindspore/dataset/vision/py_transforms_util.py @@ -24,6 +24,7 @@ import numpy as np from PIL import Image, ImageOps, ImageEnhance, __version__ from .utils import Inter +from ..core.py_util_helpers import is_numpy augment_error_message = 'img should be PIL image. Got {}. Use Decode() for encoded data or ToPIL() for decoded data.' @@ -41,39 +42,6 @@ def is_pil(img): return isinstance(img, Image.Image) -def is_numpy(img): - """ - Check if the input image is NumPy format. - - Args: - img: Image to be checked. - - Returns: - Bool, True if input is NumPy image. - """ - return isinstance(img, np.ndarray) - - -def compose(img, transforms): - """ - Compose a list of transforms and apply on the image. - - Args: - img (numpy.ndarray): An image in NumPy ndarray. - transforms (list): A list of transform Class objects to be composed. - - Returns: - img (numpy.ndarray), An augmented image in NumPy ndarray. - """ - if is_numpy(img): - for transform in transforms: - img = transform(img) - if is_numpy(img): - return img - raise TypeError('img should be NumPy ndarray. Got {}. Append ToTensor() to transforms'.format(type(img))) - raise TypeError('img should be NumPy ndarray. Got {}.'.format(type(img))) - - def normalize(img, mean, std): """ Normalize the image between [0, 1] with respect to mean and standard deviation. @@ -1221,32 +1189,6 @@ def random_affine(img, angle, translations, scale, shear, resample, fill_value=0 return img.transform(output_size, Image.AFFINE, matrix, resample, **kwargs) -def one_hot_encoding(label, num_classes, epsilon): - """ - Apply label smoothing transformation to the input label, and make label be more smoothing and continuous. - - Args: - label (numpy.ndarray): label to be applied label smoothing. - num_classes (int): Num class of object in dataset, value should over 0. - epsilon (float): The adjustable Hyper parameter. Default is 0.0. - - Returns: - img (numpy.ndarray), label after being one hot encoded and done label smoothed. - """ - if label > num_classes: - raise ValueError('the num_classes is smaller than the category number.') - - num_elements = label.size - one_hot_label = np.zeros((num_elements, num_classes), dtype=int) - - if isinstance(label, list) is False: - label = [label] - for index in range(num_elements): - one_hot_label[index, label[index]] = 1 - - return (1 - epsilon) * one_hot_label + epsilon / num_classes - - def mix_up_single(batch_size, img, label, alpha=0.2): """ Apply mix up transformation to image and label in single batch internal, One hot encoding should done before this. diff --git a/mindspore/dataset/transforms/vision/utils.py b/mindspore/dataset/vision/utils.py similarity index 100% rename from mindspore/dataset/transforms/vision/utils.py rename to mindspore/dataset/vision/utils.py diff --git a/mindspore/dataset/transforms/vision/validators.py b/mindspore/dataset/vision/validators.py similarity index 97% rename from mindspore/dataset/transforms/vision/validators.py rename to mindspore/dataset/vision/validators.py index f32e68d923..384802d9ab 100644 --- a/mindspore/dataset/transforms/vision/validators.py +++ b/mindspore/dataset/vision/validators.py @@ -19,10 +19,10 @@ from functools import wraps import numpy as np from mindspore._c_dataengine import TensorOp -from .utils import Inter, Border, ImageBatchFormat -from ...core.validator_helpers import check_value, check_uint8, FLOAT_MAX_INTEGER, check_pos_float32, \ +from mindspore.dataset.core.validator_helpers import check_value, check_uint8, FLOAT_MAX_INTEGER, check_pos_float32, \ check_2tuple, check_range, check_positive, INT32_MAX, parse_user_args, type_check, type_check_list, \ check_tensor_op, UINT8_MAX, check_value_normalize_std +from .utils import Inter, Border, ImageBatchFormat def check_crop_size(size): @@ -678,21 +678,6 @@ def check_positive_degrees(method): return new_method -def check_compose_list(method): - """Wrapper method to check the transform list of ComposeOp.""" - - @wraps(method) - def new_method(self, *args, **kwargs): - [transforms], _ = parse_user_args(method, *args, **kwargs) - - type_check(transforms, (list,), transforms) - if not transforms: - raise ValueError("transforms list is empty.") - - return method(self, *args, **kwargs) - - return new_method - def check_random_select_subpolicy_op(method): """Wrapper method to check the parameters of RandomSelectSubpolicyOp.""" diff --git a/mindspore/train/callback/_summary_collector.py b/mindspore/train/callback/_summary_collector.py index a025df6a34..ac7266075b 100644 --- a/mindspore/train/callback/_summary_collector.py +++ b/mindspore/train/callback/_summary_collector.py @@ -727,7 +727,7 @@ class SummaryCollector(Callback): Get dataset path of MindDataset object. Args: - output_dataset (Union[Dataset, ImageFolderDatasetV2, MnistDataset, Cifar10Dataset, Cifar100Dataset, + output_dataset (Union[Dataset, ImageFolderDataset, MnistDataset, Cifar10Dataset, Cifar100Dataset, VOCDataset, CelebADataset, MindDataset, ManifestDataset, TFRecordDataset, TextFileDataset]): Refer to mindspore.dataset.Dataset. @@ -738,7 +738,7 @@ class SummaryCollector(Callback): IndexError: it means get dataset path failed. """ dataset_package = import_module('mindspore.dataset') - dataset_dir_set = (dataset_package.ImageFolderDatasetV2, dataset_package.MnistDataset, + dataset_dir_set = (dataset_package.ImageFolderDataset, dataset_package.MnistDataset, dataset_package.Cifar10Dataset, dataset_package.Cifar100Dataset, dataset_package.VOCDataset, dataset_package.CelebADataset) dataset_file_set = (dataset_package.MindDataset, dataset_package.ManifestDataset) diff --git a/model_zoo/official/cv/faster_rcnn/src/dataset.py b/model_zoo/official/cv/faster_rcnn/src/dataset.py index 346ed5a6cd..47dd800457 100644 --- a/model_zoo/official/cv/faster_rcnn/src/dataset.py +++ b/model_zoo/official/cv/faster_rcnn/src/dataset.py @@ -449,7 +449,7 @@ def create_fasterrcnn_dataset(mindrecord_file, batch_size=2, repeat_num=12, devi if is_training: ds = ds.map(input_columns=["image", "annotation"], output_columns=["image", "image_shape", "box", "label", "valid_num"], - columns_order=["image", "image_shape", "box", "label", "valid_num"], + column_order=["image", "image_shape", "box", "label", "valid_num"], operations=compose_map_func, num_parallel_workers=num_parallel_workers) flip = (np.random.rand() < config.flip_ratio) @@ -467,7 +467,7 @@ def create_fasterrcnn_dataset(mindrecord_file, batch_size=2, repeat_num=12, devi else: ds = ds.map(input_columns=["image", "annotation"], output_columns=["image", "image_shape", "box", "label", "valid_num"], - columns_order=["image", "image_shape", "box", "label", "valid_num"], + column_order=["image", "image_shape", "box", "label", "valid_num"], operations=compose_map_func, num_parallel_workers=num_parallel_workers) diff --git a/model_zoo/official/cv/inceptionv3/src/dataset.py b/model_zoo/official/cv/inceptionv3/src/dataset.py index 73c84bc959..113ad56907 100644 --- a/model_zoo/official/cv/inceptionv3/src/dataset.py +++ b/model_zoo/official/cv/inceptionv3/src/dataset.py @@ -37,10 +37,10 @@ def create_dataset(dataset_path, do_train, rank, group_size, repeat_num=1): dataset """ if group_size == 1: - ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=cfg.work_nums, shuffle=True) + ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=cfg.work_nums, shuffle=True) else: - ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=cfg.work_nums, shuffle=True, - num_shards=group_size, shard_id=rank) + ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=cfg.work_nums, shuffle=True, + num_shards=group_size, shard_id=rank) # define map operations if do_train: trans = [ diff --git a/model_zoo/official/cv/maskrcnn/src/dataset.py b/model_zoo/official/cv/maskrcnn/src/dataset.py index 15d20f1e46..2330d81a5e 100644 --- a/model_zoo/official/cv/maskrcnn/src/dataset.py +++ b/model_zoo/official/cv/maskrcnn/src/dataset.py @@ -505,7 +505,7 @@ def create_maskrcnn_dataset(mindrecord_file, batch_size=2, device_num=1, rank_id if is_training: ds = ds.map(input_columns=["image", "annotation", "mask", "mask_shape"], output_columns=["image", "image_shape", "box", "label", "valid_num", "mask"], - columns_order=["image", "image_shape", "box", "label", "valid_num", "mask"], + column_order=["image", "image_shape", "box", "label", "valid_num", "mask"], operations=compose_map_func, python_multiprocessing=False, num_parallel_workers=num_parallel_workers) @@ -514,7 +514,7 @@ def create_maskrcnn_dataset(mindrecord_file, batch_size=2, device_num=1, rank_id else: ds = ds.map(input_columns=["image", "annotation", "mask", "mask_shape"], output_columns=["image", "image_shape", "box", "label", "valid_num", "mask"], - columns_order=["image", "image_shape", "box", "label", "valid_num", "mask"], + column_order=["image", "image_shape", "box", "label", "valid_num", "mask"], operations=compose_map_func, num_parallel_workers=num_parallel_workers) ds = ds.batch(batch_size, drop_remainder=True) diff --git a/model_zoo/official/cv/mobilenetv2/src/dataset.py b/model_zoo/official/cv/mobilenetv2/src/dataset.py index 1180d7b021..7e546991ef 100644 --- a/model_zoo/official/cv/mobilenetv2/src/dataset.py +++ b/model_zoo/official/cv/mobilenetv2/src/dataset.py @@ -26,6 +26,7 @@ import mindspore.dataset.engine as de import mindspore.dataset.transforms.vision.c_transforms as C import mindspore.dataset.transforms.c_transforms as C2 + def create_dataset(dataset_path, do_train, config, repeat_num=1): """ create a train or eval dataset @@ -44,20 +45,19 @@ def create_dataset(dataset_path, do_train, config, repeat_num=1): rank_size = int(os.getenv("RANK_SIZE", '1')) rank_id = int(os.getenv("RANK_ID", '0')) if rank_size == 1: - ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True) + ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True) else: - ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True, - num_shards=rank_size, shard_id=rank_id) + ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True, + num_shards=rank_size, shard_id=rank_id) elif config.platform == "GPU": if do_train: from mindspore.communication.management import get_rank, get_group_size - ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True, - num_shards=get_group_size(), shard_id=get_rank()) + ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True, + num_shards=get_group_size(), shard_id=get_rank()) else: - ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True) + ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True) elif config.platform == "CPU": - ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True) - + ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True) resize_height = config.image_height resize_width = config.image_width @@ -71,7 +71,8 @@ def create_dataset(dataset_path, do_train, config, repeat_num=1): resize_op = C.Resize((256, 256)) center_crop = C.CenterCrop(resize_width) rescale_op = C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4) - normalize_op = C.Normalize(mean=[0.485*255, 0.456*255, 0.406*255], std=[0.229*255, 0.224*255, 0.225*255]) + normalize_op = C.Normalize(mean=[0.485 * 255, 0.456 * 255, 0.406 * 255], + std=[0.229 * 255, 0.224 * 255, 0.225 * 255]) change_swap_op = C.HWC2CHW() if do_train: @@ -95,6 +96,7 @@ def create_dataset(dataset_path, do_train, config, repeat_num=1): return ds + def extract_features(net, dataset_path, config): features_folder = dataset_path + '_features' if not os.path.exists(features_folder): @@ -110,13 +112,13 @@ def extract_features(net, dataset_path, config): for data in pbar: features_path = os.path.join(features_folder, f"feature_{i}.npy") label_path = os.path.join(features_folder, f"label_{i}.npy") - if not(os.path.exists(features_path) and os.path.exists(label_path)): + if not (os.path.exists(features_path) and os.path.exists(label_path)): image = data["image"] label = data["label"] features = model.predict(Tensor(image)) np.save(features_path, features.asnumpy()) np.save(label_path, label) - pbar.set_description("Process dataset batch: %d"%(i+1)) + pbar.set_description("Process dataset batch: %d" % (i + 1)) i += 1 return step_size diff --git a/model_zoo/official/cv/mobilenetv2_quant/src/dataset.py b/model_zoo/official/cv/mobilenetv2_quant/src/dataset.py index 81f49682c8..67c48c765c 100644 --- a/model_zoo/official/cv/mobilenetv2_quant/src/dataset.py +++ b/model_zoo/official/cv/mobilenetv2_quant/src/dataset.py @@ -21,7 +21,8 @@ import mindspore.common.dtype as mstype import mindspore.dataset.engine as de import mindspore.dataset.transforms.vision.c_transforms as C import mindspore.dataset.transforms.c_transforms as C2 -import mindspore.dataset.transforms.vision.py_transforms as P +import mindspore.dataset.transforms.py_transforms +import mindspore.dataset.vision.py_transforms as P def create_dataset(dataset_path, do_train, config, device_target, repeat_num=1, batch_size=32): @@ -44,7 +45,7 @@ def create_dataset(dataset_path, do_train, config, device_target, repeat_num=1, if config.data_load_mode == "mindrecord": load_func = partial(de.MindDataset, dataset_path, columns_list) else: - load_func = partial(de.ImageFolderDatasetV2, dataset_path) + load_func = partial(de.ImageFolderDataset, dataset_path) if do_train: if rank_size == 1: ds = load_func(num_parallel_workers=8, shuffle=True) @@ -56,10 +57,10 @@ def create_dataset(dataset_path, do_train, config, device_target, repeat_num=1, elif device_target == "GPU": if do_train: from mindspore.communication.management import get_rank, get_group_size - ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True, - num_shards=get_group_size(), shard_id=get_rank()) + ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True, + num_shards=get_group_size(), shard_id=get_rank()) else: - ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True) + ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True) else: raise ValueError("Unsupported device_target.") @@ -118,12 +119,12 @@ def create_dataset_py(dataset_path, do_train, config, device_target, repeat_num= rank_id = int(os.getenv("RANK_ID")) if do_train: if rank_size == 1: - ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True) + ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True) else: - ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True, - num_shards=rank_size, shard_id=rank_id) + ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True, + num_shards=rank_size, shard_id=rank_id) else: - ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=False) + ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=False) else: raise ValueError("Unsupported device target.") @@ -149,9 +150,9 @@ def create_dataset_py(dataset_path, do_train, config, device_target, repeat_num= else: trans = [decode_op, resize_op, center_crop, to_tensor, normalize_op] - compose = P.ComposeOp(trans) + compose = mindspore.dataset.transforms.py_transforms.Compose(trans) - ds = ds.map(input_columns="image", operations=compose(), num_parallel_workers=8, python_multiprocessing=True) + ds = ds.map(input_columns="image", operations=compose, num_parallel_workers=8, python_multiprocessing=True) # apply batch operations ds = ds.batch(batch_size, drop_remainder=True) diff --git a/model_zoo/official/cv/mobilenetv3/src/dataset.py b/model_zoo/official/cv/mobilenetv3/src/dataset.py index 869c6ab3ea..7b1a355053 100644 --- a/model_zoo/official/cv/mobilenetv3/src/dataset.py +++ b/model_zoo/official/cv/mobilenetv3/src/dataset.py @@ -37,10 +37,10 @@ def create_dataset(dataset_path, do_train, config, device_target, repeat_num=1, if device_target == "GPU": if do_train: from mindspore.communication.management import get_rank, get_group_size - ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True, - num_shards=get_group_size(), shard_id=get_rank()) + ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True, + num_shards=get_group_size(), shard_id=get_rank()) else: - ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True) + ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True) else: raise ValueError("Unsupported device_target.") diff --git a/model_zoo/official/cv/nasnet/src/dataset.py b/model_zoo/official/cv/nasnet/src/dataset.py index c5e2d03033..3386408243 100755 --- a/model_zoo/official/cv/nasnet/src/dataset.py +++ b/model_zoo/official/cv/nasnet/src/dataset.py @@ -37,24 +37,24 @@ def create_dataset(dataset_path, config, do_train, repeat_num=1): rank = config.rank group_size = config.group_size if group_size == 1: - ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=config.work_nums, shuffle=True) + ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=config.work_nums, shuffle=True) else: - ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=config.work_nums, shuffle=True, - num_shards=group_size, shard_id=rank) + ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=config.work_nums, shuffle=True, + num_shards=group_size, shard_id=rank) # define map operations if do_train: trans = [ C.RandomCropDecodeResize(config.image_size), C.RandomHorizontalFlip(prob=0.5), - C.RandomColorAdjust(brightness=0.4, saturation=0.5) # fast mode - #C.RandomColorAdjust(brightness=0.4, contrast=0.5, saturation=0.5, hue=0.2) - ] + C.RandomColorAdjust(brightness=0.4, saturation=0.5) # fast mode + # C.RandomColorAdjust(brightness=0.4, contrast=0.5, saturation=0.5, hue=0.2) + ] else: trans = [ C.Decode(), - C.Resize(int(config.image_size/0.875)), + C.Resize(int(config.image_size / 0.875)), C.CenterCrop(config.image_size) - ] + ] trans += [ C.Rescale(1.0 / 255.0, 0.0), C.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]), diff --git a/model_zoo/official/cv/resnet/src/dataset.py b/model_zoo/official/cv/resnet/src/dataset.py index 86ffecb1ec..13c5a76dc9 100755 --- a/model_zoo/official/cv/resnet/src/dataset.py +++ b/model_zoo/official/cv/resnet/src/dataset.py @@ -98,10 +98,10 @@ def create_dataset2(dataset_path, do_train, repeat_num=1, batch_size=32, target= device_num = get_group_size() if device_num == 1: - ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True) + ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True) else: - ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True, - num_shards=device_num, shard_id=rank_id) + ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True, + num_shards=device_num, shard_id=rank_id) image_size = 224 mean = [0.485 * 255, 0.456 * 255, 0.406 * 255] @@ -153,10 +153,10 @@ def create_dataset3(dataset_path, do_train, repeat_num=1, batch_size=32, target= device_num, rank_id = _get_rank_info() if device_num == 1: - ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True) + ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True) else: - ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True, - num_shards=device_num, shard_id=rank_id) + ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True, + num_shards=device_num, shard_id=rank_id) image_size = 224 mean = [0.475 * 255, 0.451 * 255, 0.392 * 255] std = [0.275 * 255, 0.267 * 255, 0.278 * 255] @@ -207,10 +207,10 @@ def create_dataset4(dataset_path, do_train, repeat_num=1, batch_size=32, target= if target == "Ascend": device_num, rank_id = _get_rank_info() if device_num == 1: - ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=12, shuffle=True) + ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=12, shuffle=True) else: - ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=12, shuffle=True, - num_shards=device_num, shard_id=rank_id) + ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=12, shuffle=True, + num_shards=device_num, shard_id=rank_id) image_size = 224 mean = [123.68, 116.78, 103.94] std = [1.0, 1.0, 1.0] diff --git a/model_zoo/official/cv/resnet50_quant/src/dataset.py b/model_zoo/official/cv/resnet50_quant/src/dataset.py index 7841bc403b..5de7688f33 100755 --- a/model_zoo/official/cv/resnet50_quant/src/dataset.py +++ b/model_zoo/official/cv/resnet50_quant/src/dataset.py @@ -21,7 +21,8 @@ import mindspore.common.dtype as mstype import mindspore.dataset.engine as de import mindspore.dataset.transforms.vision.c_transforms as C import mindspore.dataset.transforms.c_transforms as C2 -import mindspore.dataset.transforms.vision.py_transforms as P +import mindspore.dataset.transforms.py_transforms +import mindspore.dataset.vision.py_transforms as P from mindspore.communication.management import init, get_rank, get_group_size from src.config import config_quant @@ -54,7 +55,7 @@ def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32, target=" if config.data_load_mode == "mindrecord": load_func = partial(de.MindDataset, dataset_path, columns_list) else: - load_func = partial(de.ImageFolderDatasetV2, dataset_path) + load_func = partial(de.ImageFolderDataset, dataset_path) if device_num == 1: ds = load_func(num_parallel_workers=8, shuffle=True) else: @@ -120,12 +121,12 @@ def create_dataset_py(dataset_path, do_train, repeat_num=1, batch_size=32, targe if do_train: if device_num == 1: - ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True) + ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True) else: - ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True, - num_shards=device_num, shard_id=rank_id) + ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True, + num_shards=device_num, shard_id=rank_id) else: - ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=False) + ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=False) image_size = 224 @@ -145,8 +146,8 @@ def create_dataset_py(dataset_path, do_train, repeat_num=1, batch_size=32, targe else: trans = [decode_op, resize_op, center_crop, to_tensor, normalize_op] - compose = P.ComposeOp(trans) - ds = ds.map(input_columns="image", operations=compose(), num_parallel_workers=8, python_multiprocessing=True) + compose = mindspore.dataset.transforms.py_transforms.Compose(trans) + ds = ds.map(input_columns="image", operations=compose, num_parallel_workers=8, python_multiprocessing=True) # apply batch operations ds = ds.batch(batch_size, drop_remainder=True) diff --git a/model_zoo/official/cv/resnet_thor/src/dataset.py b/model_zoo/official/cv/resnet_thor/src/dataset.py index dbcbf1661a..d9df510c17 100644 --- a/model_zoo/official/cv/resnet_thor/src/dataset.py +++ b/model_zoo/official/cv/resnet_thor/src/dataset.py @@ -47,10 +47,10 @@ def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32, target=" num_parallels = 4 if device_num == 1: - ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=num_parallels, shuffle=True) + ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=num_parallels, shuffle=True) else: - ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=num_parallels, shuffle=True, - num_shards=device_num, shard_id=rank_id) + ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=num_parallels, shuffle=True, + num_shards=device_num, shard_id=rank_id) image_size = 224 mean = [0.485 * 255, 0.456 * 255, 0.406 * 255] @@ -86,6 +86,7 @@ def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32, target=" return ds + def _get_rank_info(): """ get rank size and rank id diff --git a/model_zoo/official/cv/resnext50/src/dataset.py b/model_zoo/official/cv/resnext50/src/dataset.py index 66fc653c47..fd4f15f011 100644 --- a/model_zoo/official/cv/resnext50/src/dataset.py +++ b/model_zoo/official/cv/resnext50/src/dataset.py @@ -134,9 +134,9 @@ def classification_dataset(data_dir, image_size, per_batch_size, max_epoch, rank transform_label = target_transform if input_mode == 'folder': - de_dataset = de.ImageFolderDatasetV2(data_dir, num_parallel_workers=num_parallel_workers, - shuffle=shuffle, sampler=sampler, class_indexing=class_indexing, - num_shards=group_size, shard_id=rank) + de_dataset = de.ImageFolderDataset(data_dir, num_parallel_workers=num_parallel_workers, + shuffle=shuffle, sampler=sampler, class_indexing=class_indexing, + num_shards=group_size, shard_id=rank) else: dataset = TxtDataset(root, data_dir) sampler = DistributedSampler(dataset, rank, group_size, shuffle=shuffle) diff --git a/model_zoo/official/cv/shufflenetv2/src/dataset.py b/model_zoo/official/cv/shufflenetv2/src/dataset.py index f67c37f0cc..13169c1078 100644 --- a/model_zoo/official/cv/shufflenetv2/src/dataset.py +++ b/model_zoo/official/cv/shufflenetv2/src/dataset.py @@ -30,6 +30,7 @@ class toBGR(): img = np.ascontiguousarray(img) return img + def create_dataset(dataset_path, do_train, rank, group_size, repeat_num=1): """ create a train or eval dataset @@ -45,23 +46,23 @@ def create_dataset(dataset_path, do_train, rank, group_size, repeat_num=1): dataset """ if group_size == 1: - ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=cfg.work_nums, shuffle=True) + ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=cfg.work_nums, shuffle=True) else: - ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=cfg.work_nums, shuffle=True, - num_shards=group_size, shard_id=rank) + ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=cfg.work_nums, shuffle=True, + num_shards=group_size, shard_id=rank) # define map operations if do_train: trans = [ C.RandomCropDecodeResize(224), C.RandomHorizontalFlip(prob=0.5), C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4) - ] + ] else: trans = [ C.Decode(), C.Resize(256), C.CenterCrop(224) - ] + ] trans += [ toBGR(), C.Rescale(1.0 / 255.0, 0.0), diff --git a/model_zoo/official/cv/ssd/src/dataset.py b/model_zoo/official/cv/ssd/src/dataset.py index 6de9eb7012..21269a7d4b 100644 --- a/model_zoo/official/cv/ssd/src/dataset.py +++ b/model_zoo/official/cv/ssd/src/dataset.py @@ -403,7 +403,7 @@ def create_ssd_dataset(mindrecord_file, batch_size=32, repeat_num=10, device_num output_columns = ["img_id", "image", "image_shape"] trans = [normalize_op, change_swap_op] ds = ds.map(input_columns=["img_id", "image", "annotation"], - output_columns=output_columns, columns_order=output_columns, + output_columns=output_columns, column_order=output_columns, operations=compose_map_func, python_multiprocessing=is_training, num_parallel_workers=num_parallel_workers) ds = ds.map(input_columns=["image"], operations=trans, python_multiprocessing=is_training, diff --git a/model_zoo/official/cv/vgg16/src/dataset.py b/model_zoo/official/cv/vgg16/src/dataset.py index c510b49497..0f6d538893 100644 --- a/model_zoo/official/cv/vgg16/src/dataset.py +++ b/model_zoo/official/cv/vgg16/src/dataset.py @@ -149,9 +149,9 @@ def classification_dataset(data_dir, image_size, per_batch_size, rank=0, group_s transform_label = target_transform if input_mode == 'folder': - de_dataset = de.ImageFolderDatasetV2(data_dir, num_parallel_workers=num_parallel_workers, - shuffle=shuffle, sampler=sampler, class_indexing=class_indexing, - num_shards=group_size, shard_id=rank) + de_dataset = de.ImageFolderDataset(data_dir, num_parallel_workers=num_parallel_workers, + shuffle=shuffle, sampler=sampler, class_indexing=class_indexing, + num_shards=group_size, shard_id=rank) else: dataset = TxtDataset(root, data_dir) sampler = DistributedSampler(dataset, rank, group_size, shuffle=shuffle) diff --git a/model_zoo/official/cv/yolov3_darknet53/src/yolo_dataset.py b/model_zoo/official/cv/yolov3_darknet53/src/yolo_dataset.py index 8045c468f2..947ab6136c 100644 --- a/model_zoo/official/cv/yolov3_darknet53/src/yolo_dataset.py +++ b/model_zoo/official/cv/yolov3_darknet53/src/yolo_dataset.py @@ -178,7 +178,7 @@ def create_yolo_dataset(image_dir, anno_path, batch_size, max_epoch, device_num, compose_map_func = (lambda image, img_id: reshape_fn(image, img_id, config)) ds = ds.map(input_columns=["image", "img_id"], output_columns=["image", "image_shape", "img_id"], - columns_order=["image", "image_shape", "img_id"], + column_order=["image", "image_shape", "img_id"], operations=compose_map_func, num_parallel_workers=8) ds = ds.map(input_columns=["image"], operations=hwc_to_chw, num_parallel_workers=8) ds = ds.batch(batch_size, drop_remainder=True) diff --git a/model_zoo/official/cv/yolov3_darknet53_quant/src/yolo_dataset.py b/model_zoo/official/cv/yolov3_darknet53_quant/src/yolo_dataset.py index 45657db823..af376e43cb 100644 --- a/model_zoo/official/cv/yolov3_darknet53_quant/src/yolo_dataset.py +++ b/model_zoo/official/cv/yolov3_darknet53_quant/src/yolo_dataset.py @@ -175,7 +175,7 @@ def create_yolo_dataset(image_dir, anno_path, batch_size, max_epoch, device_num, compose_map_func = (lambda image, img_id: reshape_fn(image, img_id, config)) ds = ds.map(input_columns=["image", "img_id"], output_columns=["image", "image_shape", "img_id"], - columns_order=["image", "image_shape", "img_id"], + column_order=["image", "image_shape", "img_id"], operations=compose_map_func, num_parallel_workers=8) ds = ds.map(input_columns=["image"], operations=hwc_to_chw, num_parallel_workers=8) ds = ds.batch(batch_size, drop_remainder=True) diff --git a/model_zoo/official/cv/yolov3_resnet18/src/dataset.py b/model_zoo/official/cv/yolov3_resnet18/src/dataset.py index 7c5177a3fe..0bfcc74bf1 100644 --- a/model_zoo/official/cv/yolov3_resnet18/src/dataset.py +++ b/model_zoo/official/cv/yolov3_resnet18/src/dataset.py @@ -303,7 +303,7 @@ def create_yolo_dataset(mindrecord_dir, batch_size=32, repeat_num=1, device_num= hwc_to_chw = C.HWC2CHW() ds = ds.map(input_columns=["image", "annotation"], output_columns=["image", "bbox_1", "bbox_2", "bbox_3", "gt_box1", "gt_box2", "gt_box3"], - columns_order=["image", "bbox_1", "bbox_2", "bbox_3", "gt_box1", "gt_box2", "gt_box3"], + column_order=["image", "bbox_1", "bbox_2", "bbox_3", "gt_box1", "gt_box2", "gt_box3"], operations=compose_map_func, num_parallel_workers=num_parallel_workers) ds = ds.map(input_columns=["image"], operations=hwc_to_chw, num_parallel_workers=num_parallel_workers) ds = ds.batch(batch_size, drop_remainder=True) @@ -311,6 +311,6 @@ def create_yolo_dataset(mindrecord_dir, batch_size=32, repeat_num=1, device_num= else: ds = ds.map(input_columns=["image", "annotation"], output_columns=["image", "image_shape", "annotation"], - columns_order=["image", "image_shape", "annotation"], + column_order=["image", "image_shape", "annotation"], operations=compose_map_func, num_parallel_workers=num_parallel_workers) return ds diff --git a/model_zoo/official/nlp/bert/src/clue_classification_dataset_process.py b/model_zoo/official/nlp/bert/src/clue_classification_dataset_process.py index 042bc0a9c6..0d2201cfe6 100755 --- a/model_zoo/official/nlp/bert/src/clue_classification_dataset_process.py +++ b/model_zoo/official/nlp/bert/src/clue_classification_dataset_process.py @@ -43,7 +43,7 @@ def process_tnews_clue_dataset(data_dir, label_list, bert_vocab_path, data_usage ### Processing label if data_usage == 'test': dataset = dataset.map(input_columns=["id"], output_columns=["id", "label_id"], - columns_order=["id", "label_id", "sentence"], operations=ops.Duplicate()) + column_order=["id", "label_id", "sentence"], operations=ops.Duplicate()) dataset = dataset.map(input_columns=["label_id"], operations=ops.Fill(0)) else: label_vocab = text.Vocab.from_list(label_list) @@ -61,10 +61,10 @@ def process_tnews_clue_dataset(data_dir, label_list, bert_vocab_path, data_usage dataset = dataset.map(input_columns=["sentence"], output_columns=["text_ids"], operations=lookup) dataset = dataset.map(input_columns=["text_ids"], operations=ops.PadEnd([max_seq_len], 0)) dataset = dataset.map(input_columns=["text_ids"], output_columns=["text_ids", "mask_ids"], - columns_order=["text_ids", "mask_ids", "label_id"], operations=ops.Duplicate()) + column_order=["text_ids", "mask_ids", "label_id"], operations=ops.Duplicate()) dataset = dataset.map(input_columns=["mask_ids"], operations=ops.Mask(ops.Relational.NE, 0, mstype.int32)) dataset = dataset.map(input_columns=["text_ids"], output_columns=["text_ids", "segment_ids"], - columns_order=["text_ids", "mask_ids", "segment_ids", "label_id"], operations=ops.Duplicate()) + column_order=["text_ids", "mask_ids", "segment_ids", "label_id"], operations=ops.Duplicate()) dataset = dataset.map(input_columns=["segment_ids"], operations=ops.Fill(0)) dataset = dataset.batch(batch_size, drop_remainder=drop_remainder) return dataset @@ -87,7 +87,7 @@ def process_cmnli_clue_dataset(data_dir, label_list, bert_vocab_path, data_usage ### Processing label if data_usage == 'test': dataset = dataset.map(input_columns=["id"], output_columns=["id", "label_id"], - columns_order=["id", "label_id", "sentence1", "sentence2"], operations=ops.Duplicate()) + column_order=["id", "label_id", "sentence1", "sentence2"], operations=ops.Duplicate()) dataset = dataset.map(input_columns=["label_id"], operations=ops.Fill(0)) else: label_vocab = text.Vocab.from_list(label_list) @@ -110,26 +110,26 @@ def process_cmnli_clue_dataset(data_dir, label_list, bert_vocab_path, data_usage operations=ops.Concatenate(append=np.array(["[SEP]"], dtype='S'))) ### Generating segment_ids dataset = dataset.map(input_columns=["sentence1"], output_columns=["sentence1", "type_sentence1"], - columns_order=["sentence1", "type_sentence1", "sentence2", "label_id"], + column_order=["sentence1", "type_sentence1", "sentence2", "label_id"], operations=ops.Duplicate()) dataset = dataset.map(input_columns=["sentence2"], output_columns=["sentence2", "type_sentence2"], - columns_order=["sentence1", "type_sentence1", "sentence2", "type_sentence2", "label_id"], + column_order=["sentence1", "type_sentence1", "sentence2", "type_sentence2", "label_id"], operations=ops.Duplicate()) dataset = dataset.map(input_columns=["type_sentence1"], operations=[lookup, ops.Fill(0)]) dataset = dataset.map(input_columns=["type_sentence2"], operations=[lookup, ops.Fill(1)]) dataset = dataset.map(input_columns=["type_sentence1", "type_sentence2"], output_columns=["segment_ids"], - columns_order=["sentence1", "sentence2", "segment_ids", "label_id"], + column_order=["sentence1", "sentence2", "segment_ids", "label_id"], operations=ops.Concatenate()) dataset = dataset.map(input_columns=["segment_ids"], operations=ops.PadEnd([max_seq_len], 0)) ### Generating text_ids dataset = dataset.map(input_columns=["sentence1", "sentence2"], output_columns=["text_ids"], - columns_order=["text_ids", "segment_ids", "label_id"], + column_order=["text_ids", "segment_ids", "label_id"], operations=ops.Concatenate()) dataset = dataset.map(input_columns=["text_ids"], operations=lookup) dataset = dataset.map(input_columns=["text_ids"], operations=ops.PadEnd([max_seq_len], 0)) ### Generating mask_ids dataset = dataset.map(input_columns=["text_ids"], output_columns=["text_ids", "mask_ids"], - columns_order=["text_ids", "mask_ids", "segment_ids", "label_id"], operations=ops.Duplicate()) + column_order=["text_ids", "mask_ids", "segment_ids", "label_id"], operations=ops.Duplicate()) dataset = dataset.map(input_columns=["mask_ids"], operations=ops.Mask(ops.Relational.NE, 0, mstype.int32)) dataset = dataset.batch(batch_size, drop_remainder=drop_remainder) return dataset diff --git a/model_zoo/official/recommend/deepfm/src/dataset.py b/model_zoo/official/recommend/deepfm/src/dataset.py index e023786f42..e823b05e9b 100644 --- a/model_zoo/official/recommend/deepfm/src/dataset.py +++ b/model_zoo/official/recommend/deepfm/src/dataset.py @@ -213,7 +213,7 @@ def _get_mindrecord_dataset(directory, train_mode=True, epochs=1, batch_size=100 np.array(y).flatten().reshape(batch_size, 39), np.array(z).flatten().reshape(batch_size, 1))), input_columns=['feat_ids', 'feat_vals', 'label'], - columns_order=['feat_ids', 'feat_vals', 'label'], + column_order=['feat_ids', 'feat_vals', 'label'], num_parallel_workers=8) ds = ds.repeat(epochs) return ds @@ -261,7 +261,7 @@ def _get_tf_dataset(directory, train_mode=True, epochs=1, batch_size=1000, np.array(y).flatten().reshape(batch_size, 39), np.array(z).flatten().reshape(batch_size, 1))), input_columns=['feat_ids', 'feat_vals', 'label'], - columns_order=['feat_ids', 'feat_vals', 'label'], + column_order=['feat_ids', 'feat_vals', 'label'], num_parallel_workers=8) ds = ds.repeat(epochs) return ds diff --git a/model_zoo/official/recommend/wide_and_deep/src/datasets.py b/model_zoo/official/recommend/wide_and_deep/src/datasets.py index 1a28ac5327..7555d61eac 100644 --- a/model_zoo/official/recommend/wide_and_deep/src/datasets.py +++ b/model_zoo/official/recommend/wide_and_deep/src/datasets.py @@ -230,7 +230,7 @@ def _get_tf_dataset(data_dir, train_mode=True, epochs=1, batch_size=1000, ds = ds.map(operations=_padding_func(batch_size, manual_shape, target_column), input_columns=['feat_ids', 'feat_vals', 'label'], - columns_order=['feat_ids', 'feat_vals', 'label'], num_parallel_workers=8) + column_order=['feat_ids', 'feat_vals', 'label'], num_parallel_workers=8) # if train_mode: ds = ds.repeat(epochs) return ds @@ -270,7 +270,7 @@ def _get_mindrecord_dataset(directory, train_mode=True, epochs=1, batch_size=100 ds = ds.batch(int(batch_size / line_per_sample), drop_remainder=True) ds = ds.map(_padding_func(batch_size, manual_shape, target_column), input_columns=['feat_ids', 'feat_vals', 'label'], - columns_order=['feat_ids', 'feat_vals', 'label'], + column_order=['feat_ids', 'feat_vals', 'label'], num_parallel_workers=8) ds = ds.repeat(epochs) return ds diff --git a/model_zoo/official/recommend/wide_and_deep_multitable/src/datasets.py b/model_zoo/official/recommend/wide_and_deep_multitable/src/datasets.py index 29c04a0195..2c2d553b35 100644 --- a/model_zoo/official/recommend/wide_and_deep_multitable/src/datasets.py +++ b/model_zoo/official/recommend/wide_and_deep_multitable/src/datasets.py @@ -263,7 +263,7 @@ def _get_tf_dataset(data_dir, 'multi_doc_ad_topic_id_mask', 'ad_id', 'display_ad_and_is_leak', 'display_id', 'is_leak' ], - columns_order=[ + column_order=[ 'label', 'continue_val', 'indicator_id', 'emb_128_id', 'emb_64_single_id', 'multi_doc_ad_category_id', 'multi_doc_ad_category_id_mask', 'multi_doc_event_entity_id', diff --git a/tests/st/mem_reuse/resnet_cifar_memreuse.py b/tests/st/mem_reuse/resnet_cifar_memreuse.py index fc4cf42b51..a7ca83cbf7 100644 --- a/tests/st/mem_reuse/resnet_cifar_memreuse.py +++ b/tests/st/mem_reuse/resnet_cifar_memreuse.py @@ -22,7 +22,7 @@ import mindspore.common.dtype as mstype import mindspore.context as context import mindspore.dataset as de import mindspore.dataset.transforms.c_transforms as C -import mindspore.dataset.transforms.vision.c_transforms as vision +import mindspore.dataset.vision.c_transforms as vision import mindspore.nn as nn from mindspore import Tensor from mindspore.communication.management import init diff --git a/tests/st/mem_reuse/resnet_cifar_normal.py b/tests/st/mem_reuse/resnet_cifar_normal.py index 83241220cd..e5746a254d 100644 --- a/tests/st/mem_reuse/resnet_cifar_normal.py +++ b/tests/st/mem_reuse/resnet_cifar_normal.py @@ -22,7 +22,7 @@ import mindspore.common.dtype as mstype import mindspore.context as context import mindspore.dataset as de import mindspore.dataset.transforms.c_transforms as C -import mindspore.dataset.transforms.vision.c_transforms as vision +import mindspore.dataset.vision.c_transforms as vision import mindspore.nn as nn from mindspore import Tensor from mindspore.communication.management import init diff --git a/tests/st/model_zoo_tests/wide_and_deep/python_file_for_ci/datasets.py b/tests/st/model_zoo_tests/wide_and_deep/python_file_for_ci/datasets.py index bad2106fbf..d0a5234d9a 100644 --- a/tests/st/model_zoo_tests/wide_and_deep/python_file_for_ci/datasets.py +++ b/tests/st/model_zoo_tests/wide_and_deep/python_file_for_ci/datasets.py @@ -57,7 +57,7 @@ def _get_tf_dataset(data_dir, train_mode=True, epochs=1, batch_size=1000, np.array(y).flatten().reshape(batch_size, 39), np.array(z).flatten().reshape(batch_size, 1))), input_columns=['feat_ids', 'feat_vals', 'label'], - columns_order=['feat_ids', 'feat_vals', 'label'], num_parallel_workers=8) + column_order=['feat_ids', 'feat_vals', 'label'], num_parallel_workers=8) #if train_mode: ds = ds.repeat(epochs) return ds @@ -97,7 +97,7 @@ def _get_mindrecord_dataset(directory, train_mode=True, epochs=1, batch_size=100 np.array(y).flatten().reshape(batch_size, 39), np.array(z).flatten().reshape(batch_size, 1))), input_columns=['feat_ids', 'feat_vals', 'label'], - columns_order=['feat_ids', 'feat_vals', 'label'], + column_order=['feat_ids', 'feat_vals', 'label'], num_parallel_workers=8) ds = ds.repeat(epochs) return ds diff --git a/tests/st/model_zoo_tests/yolov3/src/dataset.py b/tests/st/model_zoo_tests/yolov3/src/dataset.py index e13802566b..712fe9bb4b 100644 --- a/tests/st/model_zoo_tests/yolov3/src/dataset.py +++ b/tests/st/model_zoo_tests/yolov3/src/dataset.py @@ -22,7 +22,7 @@ from matplotlib.colors import rgb_to_hsv, hsv_to_rgb from PIL import Image import mindspore.dataset as de from mindspore.mindrecord import FileWriter -import mindspore.dataset.transforms.vision.c_transforms as C +import mindspore.dataset.vision.c_transforms as C from src.config import ConfigYOLOV3ResNet18 iter_cnt = 0 @@ -305,7 +305,7 @@ def create_yolo_dataset(mindrecord_dir, batch_size=32, repeat_num=10, device_num hwc_to_chw = C.HWC2CHW() ds = ds.map(input_columns=["image", "annotation"], output_columns=["image", "bbox_1", "bbox_2", "bbox_3", "gt_box1", "gt_box2", "gt_box3"], - columns_order=["image", "bbox_1", "bbox_2", "bbox_3", "gt_box1", "gt_box2", "gt_box3"], + column_order=["image", "bbox_1", "bbox_2", "bbox_3", "gt_box1", "gt_box2", "gt_box3"], operations=compose_map_func, num_parallel_workers=num_parallel_workers) ds = ds.map(input_columns=["image"], operations=hwc_to_chw, num_parallel_workers=num_parallel_workers) ds = ds.batch(batch_size, drop_remainder=True) @@ -313,6 +313,6 @@ def create_yolo_dataset(mindrecord_dir, batch_size=32, repeat_num=10, device_num else: ds = ds.map(input_columns=["image", "annotation"], output_columns=["image", "image_shape", "annotation"], - columns_order=["image", "image_shape", "annotation"], + column_order=["image", "image_shape", "annotation"], operations=compose_map_func, num_parallel_workers=num_parallel_workers) return ds diff --git a/tests/st/networks/models/deeplabv3/src/md_dataset.py b/tests/st/networks/models/deeplabv3/src/md_dataset.py index e136da23e1..4f5adc9890 100644 --- a/tests/st/networks/models/deeplabv3/src/md_dataset.py +++ b/tests/st/networks/models/deeplabv3/src/md_dataset.py @@ -15,7 +15,7 @@ """Dataset module.""" from PIL import Image import mindspore.dataset as de -import mindspore.dataset.transforms.vision.c_transforms as C +import mindspore.dataset.vision.c_transforms as C import numpy as np from .ei_dataset import HwVocRawDataset diff --git a/tests/st/networks/models/resnet50/src/dataset.py b/tests/st/networks/models/resnet50/src/dataset.py index ae15f4159e..a5f3f61ad5 100755 --- a/tests/st/networks/models/resnet50/src/dataset.py +++ b/tests/st/networks/models/resnet50/src/dataset.py @@ -18,7 +18,7 @@ import os import mindspore.common.dtype as mstype import mindspore.dataset.engine as de -import mindspore.dataset.transforms.vision.c_transforms as C +import mindspore.dataset.vision.c_transforms as C import mindspore.dataset.transforms.c_transforms as C2 @@ -39,10 +39,10 @@ def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32): device_num = int(os.getenv("RANK_SIZE")) rank_id = int(os.getenv("RANK_ID")) if device_num == 1: - ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True) + ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True) else: - ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True, - num_shards=device_num, shard_id=rank_id) + ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True, + num_shards=device_num, shard_id=rank_id) image_size = 224 mean = [0.485 * 255, 0.456 * 255, 0.406 * 255] diff --git a/tests/st/networks/models/resnet50/src_thor/dataset.py b/tests/st/networks/models/resnet50/src_thor/dataset.py index 091172e62c..b63e0a0de7 100644 --- a/tests/st/networks/models/resnet50/src_thor/dataset.py +++ b/tests/st/networks/models/resnet50/src_thor/dataset.py @@ -21,7 +21,7 @@ import mindspore.common.dtype as mstype import mindspore.dataset as dataset import mindspore.dataset.engine as de import mindspore.dataset.transforms.c_transforms as C2 -import mindspore.dataset.transforms.vision.c_transforms as C +import mindspore.dataset.vision.c_transforms as C dataset.config.set_seed(1) @@ -43,10 +43,10 @@ def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32): device_num = int(os.getenv("RANK_SIZE")) rank_id = int(os.getenv("RANK_ID")) if device_num == 1: - ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True) + ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True) else: - ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True, - num_shards=device_num, shard_id=rank_id) + ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True, + num_shards=device_num, shard_id=rank_id) image_size = 224 mean = [0.485 * 255, 0.456 * 255, 0.406 * 255] diff --git a/tests/st/networks/test_gpu_lenet.py b/tests/st/networks/test_gpu_lenet.py index ad77a69170..77f42cb9f2 100644 --- a/tests/st/networks/test_gpu_lenet.py +++ b/tests/st/networks/test_gpu_lenet.py @@ -21,11 +21,11 @@ import pytest import mindspore.context as context import mindspore.dataset as ds import mindspore.dataset.transforms.c_transforms as C -import mindspore.dataset.transforms.vision.c_transforms as CV +import mindspore.dataset.vision.c_transforms as CV import mindspore.nn as nn from mindspore import Tensor from mindspore.common import dtype as mstype -from mindspore.dataset.transforms.vision import Inter +from mindspore.dataset.vision import Inter from mindspore.nn import Dense, TrainOneStepCell, WithLossCell from mindspore.nn.metrics import Accuracy from mindspore.nn.optim import Momentum diff --git a/tests/st/ops/ascend/test_tdt_data_ms.py b/tests/st/ops/ascend/test_tdt_data_ms.py index b4fae1c2d3..c0e1bf69aa 100644 --- a/tests/st/ops/ascend/test_tdt_data_ms.py +++ b/tests/st/ops/ascend/test_tdt_data_ms.py @@ -17,11 +17,11 @@ import numpy as np import mindspore.context as context import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.c_transforms as vision +import mindspore.dataset.vision.c_transforms as vision import mindspore.nn as nn from mindspore.common.api import _executor from mindspore.common.tensor import Tensor -from mindspore.dataset.transforms.vision import Inter +from mindspore.dataset.vision import Inter from mindspore.ops import operations as P context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") @@ -83,8 +83,6 @@ if __name__ == '__main__': class dataiter(nn.Cell): - def __init__(self): - super(dataiter, self).__init__() def construct(self): input_, _ = get_next() diff --git a/tests/st/probability/dataset.py b/tests/st/probability/dataset.py index cef6973483..dca3a6af8a 100644 --- a/tests/st/probability/dataset.py +++ b/tests/st/probability/dataset.py @@ -17,9 +17,9 @@ Produce the dataset """ import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.c_transforms as CV +import mindspore.dataset.vision.c_transforms as CV import mindspore.dataset.transforms.c_transforms as C -from mindspore.dataset.transforms.vision import Inter +from mindspore.dataset.vision import Inter from mindspore.common import dtype as mstype diff --git a/tests/st/probability/test_gpu_svi_cvae.py b/tests/st/probability/test_gpu_svi_cvae.py index 0bcf0fdadf..09700a14ec 100644 --- a/tests/st/probability/test_gpu_svi_cvae.py +++ b/tests/st/probability/test_gpu_svi_cvae.py @@ -16,7 +16,7 @@ import os import mindspore.common.dtype as mstype import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.c_transforms as CV +import mindspore.dataset.vision.c_transforms as CV import mindspore.nn as nn from mindspore import context, Tensor from mindspore.ops import operations as P diff --git a/tests/st/probability/test_gpu_svi_vae.py b/tests/st/probability/test_gpu_svi_vae.py index 6e1b23ee1b..b5ac6537c1 100644 --- a/tests/st/probability/test_gpu_svi_vae.py +++ b/tests/st/probability/test_gpu_svi_vae.py @@ -16,7 +16,7 @@ import os import mindspore.common.dtype as mstype import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.c_transforms as CV +import mindspore.dataset.vision.c_transforms as CV import mindspore.nn as nn from mindspore import context, Tensor from mindspore.ops import operations as P diff --git a/tests/st/probability/test_gpu_vae_gan.py b/tests/st/probability/test_gpu_vae_gan.py index 7a1108c715..b6bff57635 100644 --- a/tests/st/probability/test_gpu_vae_gan.py +++ b/tests/st/probability/test_gpu_vae_gan.py @@ -18,7 +18,7 @@ The VAE interface can be called to construct VAE-GAN network. import os import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.c_transforms as CV +import mindspore.dataset.vision.c_transforms as CV import mindspore.nn as nn from mindspore import context from mindspore.ops import operations as P diff --git a/tests/st/probability/test_uncertainty.py b/tests/st/probability/test_uncertainty.py index 92850141eb..c36b93f9e1 100644 --- a/tests/st/probability/test_uncertainty.py +++ b/tests/st/probability/test_uncertainty.py @@ -15,12 +15,12 @@ """ test uncertainty toolbox """ import mindspore.dataset as ds import mindspore.dataset.transforms.c_transforms as C -import mindspore.dataset.transforms.vision.c_transforms as CV +import mindspore.dataset.vision.c_transforms as CV import mindspore.nn as nn from mindspore import context, Tensor from mindspore.common import dtype as mstype from mindspore.common.initializer import TruncatedNormal -from mindspore.dataset.transforms.vision import Inter +from mindspore.dataset.vision import Inter from mindspore.nn.probability.toolbox.uncertainty_evaluation import UncertaintyEvaluation from mindspore.train.serialization import load_checkpoint, load_param_into_net diff --git a/tests/st/ps/full_ps/test_full_ps_lenet.py b/tests/st/ps/full_ps/test_full_ps_lenet.py index 2ee3caef94..7693924c8c 100644 --- a/tests/st/ps/full_ps/test_full_ps_lenet.py +++ b/tests/st/ps/full_ps/test_full_ps_lenet.py @@ -19,10 +19,10 @@ import argparse import mindspore.context as context import mindspore.dataset as ds import mindspore.dataset.transforms.c_transforms as C -import mindspore.dataset.transforms.vision.c_transforms as CV +import mindspore.dataset.vision.c_transforms as CV import mindspore.nn as nn from mindspore.common import dtype as mstype -from mindspore.dataset.transforms.vision import Inter +from mindspore.dataset.vision import Inter from mindspore.nn.metrics import Accuracy from mindspore.train import Model from mindspore.train.callback import LossMonitor diff --git a/tests/st/pynative/test_pynative_resnet50.py b/tests/st/pynative/test_pynative_resnet50.py index 6aa47dde4d..5122ce3349 100644 --- a/tests/st/pynative/test_pynative_resnet50.py +++ b/tests/st/pynative/test_pynative_resnet50.py @@ -21,7 +21,7 @@ import pytest import mindspore.common.dtype as mstype import mindspore.dataset as ds import mindspore.dataset.transforms.c_transforms as C -import mindspore.dataset.transforms.vision.c_transforms as vision +import mindspore.dataset.vision.c_transforms as vision import mindspore.nn as nn import mindspore.ops.functional as F diff --git a/tests/st/quantization/lenet_quant/dataset.py b/tests/st/quantization/lenet_quant/dataset.py index cef6973483..dca3a6af8a 100644 --- a/tests/st/quantization/lenet_quant/dataset.py +++ b/tests/st/quantization/lenet_quant/dataset.py @@ -17,9 +17,9 @@ Produce the dataset """ import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.c_transforms as CV +import mindspore.dataset.vision.c_transforms as CV import mindspore.dataset.transforms.c_transforms as C -from mindspore.dataset.transforms.vision import Inter +from mindspore.dataset.vision import Inter from mindspore.common import dtype as mstype diff --git a/tests/st/summary/test_summary.py b/tests/st/summary/test_summary.py index 7aa5d95358..1989c93e93 100644 --- a/tests/st/summary/test_summary.py +++ b/tests/st/summary/test_summary.py @@ -25,8 +25,8 @@ from mindspore import nn, Tensor, context from mindspore.nn.metrics import Accuracy from mindspore.nn.optim import Momentum from mindspore.dataset.transforms import c_transforms as C -from mindspore.dataset.transforms.vision import c_transforms as CV -from mindspore.dataset.transforms.vision import Inter +from mindspore.dataset.vision import c_transforms as CV +from mindspore.dataset.vision import Inter from mindspore.common import dtype as mstype from mindspore.common.initializer import TruncatedNormal from mindspore.ops import operations as P diff --git a/tests/st/tbe_networks/resnet_cifar.py b/tests/st/tbe_networks/resnet_cifar.py index 909cf18298..ac835052f6 100644 --- a/tests/st/tbe_networks/resnet_cifar.py +++ b/tests/st/tbe_networks/resnet_cifar.py @@ -24,7 +24,7 @@ from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMoni from mindspore.train.serialization import load_checkpoint, load_param_into_net import mindspore.dataset as ds import mindspore.dataset.transforms.c_transforms as C -import mindspore.dataset.transforms.vision.c_transforms as vision +import mindspore.dataset.vision.c_transforms as vision import mindspore.nn as nn from mindspore import Tensor from mindspore import context diff --git a/tests/st/tbe_networks/test_resnet_cifar_1p.py b/tests/st/tbe_networks/test_resnet_cifar_1p.py index 8ef48b8774..de4f93e2b5 100644 --- a/tests/st/tbe_networks/test_resnet_cifar_1p.py +++ b/tests/st/tbe_networks/test_resnet_cifar_1p.py @@ -21,7 +21,7 @@ from resnet import resnet50 import mindspore.common.dtype as mstype import mindspore.dataset as ds import mindspore.dataset.transforms.c_transforms as C -import mindspore.dataset.transforms.vision.c_transforms as vision +import mindspore.dataset.vision.c_transforms as vision import mindspore.nn as nn import mindspore.ops.functional as F from mindspore import Tensor diff --git a/tests/st/tbe_networks/test_resnet_cifar_8p.py b/tests/st/tbe_networks/test_resnet_cifar_8p.py index 5b610cceba..482be8dca5 100644 --- a/tests/st/tbe_networks/test_resnet_cifar_8p.py +++ b/tests/st/tbe_networks/test_resnet_cifar_8p.py @@ -22,7 +22,7 @@ from resnet import resnet50 import mindspore.common.dtype as mstype import mindspore.dataset as ds import mindspore.dataset.transforms.c_transforms as C -import mindspore.dataset.transforms.vision.c_transforms as vision +import mindspore.dataset.vision.c_transforms as vision import mindspore.nn as nn import mindspore.ops.functional as F from mindspore import Tensor diff --git a/tests/ut/python/dataset/test_HWC2CHW.py b/tests/ut/python/dataset/test_HWC2CHW.py index b14d0a9990..5b173fd319 100644 --- a/tests/ut/python/dataset/test_HWC2CHW.py +++ b/tests/ut/python/dataset/test_HWC2CHW.py @@ -17,8 +17,9 @@ Testing HWC2CHW op in DE """ import numpy as np import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.c_transforms as c_vision -import mindspore.dataset.transforms.vision.py_transforms as py_vision +import mindspore.dataset.transforms.py_transforms +import mindspore.dataset.vision.c_transforms as c_vision +import mindspore.dataset.vision.py_transforms as py_vision from mindspore import log as logger from util import diff_mse, visualize_list, save_and_check_md5 @@ -99,8 +100,8 @@ def test_HWC2CHW_comp(plot=False): py_vision.ToTensor(), py_vision.HWC2CHW() ] - transform = py_vision.ComposeOp(transforms) - data2 = data2.map(input_columns=["image"], operations=transform()) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) + data2 = data2.map(input_columns=["image"], operations=transform) image_c_transposed = [] image_py_transposed = [] diff --git a/tests/ut/python/dataset/test_apply.py b/tests/ut/python/dataset/test_apply.py index abc5511830..38c61db5ae 100644 --- a/tests/ut/python/dataset/test_apply.py +++ b/tests/ut/python/dataset/test_apply.py @@ -15,7 +15,7 @@ import numpy as np import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.c_transforms as vision +import mindspore.dataset.vision.c_transforms as vision from mindspore import log as logger DATA_DIR = "../data/dataset/testPK/data" @@ -46,8 +46,8 @@ def test_apply_generator_case(): def test_apply_imagefolder_case(): # apply dataset map operations - data1 = ds.ImageFolderDatasetV2(DATA_DIR, num_shards=4, shard_id=3) - data2 = ds.ImageFolderDatasetV2(DATA_DIR, num_shards=4, shard_id=3) + data1 = ds.ImageFolderDataset(DATA_DIR, num_shards=4, shard_id=3) + data2 = ds.ImageFolderDataset(DATA_DIR, num_shards=4, shard_id=3) decode_op = vision.Decode() normalize_op = vision.Normalize([121.0, 115.0, 100.0], [70.0, 68.0, 71.0]) diff --git a/tests/ut/python/dataset/test_autocontrast.py b/tests/ut/python/dataset/test_autocontrast.py index c657c74296..8c0bf951e4 100644 --- a/tests/ut/python/dataset/test_autocontrast.py +++ b/tests/ut/python/dataset/test_autocontrast.py @@ -17,8 +17,9 @@ Testing AutoContrast op in DE """ import numpy as np import mindspore.dataset.engine as de -import mindspore.dataset.transforms.vision.py_transforms as F -import mindspore.dataset.transforms.vision.c_transforms as C +import mindspore.dataset.transforms.py_transforms +import mindspore.dataset.vision.py_transforms as F +import mindspore.dataset.vision.c_transforms as C from mindspore import log as logger from util import visualize_list, visualize_one_channel_dataset, diff_mse, save_and_check_md5 @@ -35,14 +36,14 @@ def test_auto_contrast_py(plot=False): logger.info("Test AutoContrast Python Op") # Original Images - ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) - transforms_original = F.ComposeOp([F.Decode(), - F.Resize((224, 224)), - F.ToTensor()]) + transforms_original = mindspore.dataset.transforms.py_transforms.Compose([F.Decode(), + F.Resize((224, 224)), + F.ToTensor()]) ds_original = ds.map(input_columns="image", - operations=transforms_original()) + operations=transforms_original) ds_original = ds_original.batch(512) @@ -55,15 +56,16 @@ def test_auto_contrast_py(plot=False): axis=0) # AutoContrast Images - ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) - transforms_auto_contrast = F.ComposeOp([F.Decode(), - F.Resize((224, 224)), - F.AutoContrast(cutoff=10.0, ignore=[10, 20]), - F.ToTensor()]) + transforms_auto_contrast = \ + mindspore.dataset.transforms.py_transforms.Compose([F.Decode(), + F.Resize((224, 224)), + F.AutoContrast(cutoff=10.0, ignore=[10, 20]), + F.ToTensor()]) ds_auto_contrast = ds.map(input_columns="image", - operations=transforms_auto_contrast()) + operations=transforms_auto_contrast) ds_auto_contrast = ds_auto_contrast.batch(512) @@ -96,15 +98,15 @@ def test_auto_contrast_c(plot=False): logger.info("Test AutoContrast C Op") # AutoContrast Images - ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) ds = ds.map(input_columns=["image"], operations=[C.Decode(), C.Resize((224, 224))]) python_op = F.AutoContrast(cutoff=10.0, ignore=[10, 20]) c_op = C.AutoContrast(cutoff=10.0, ignore=[10, 20]) - transforms_op = F.ComposeOp([lambda img: F.ToPIL()(img.astype(np.uint8)), - python_op, - np.array])() + transforms_op = mindspore.dataset.transforms.py_transforms.Compose([lambda img: F.ToPIL()(img.astype(np.uint8)), + python_op, + np.array]) ds_auto_contrast_py = ds.map(input_columns="image", operations=transforms_op) @@ -119,7 +121,7 @@ def test_auto_contrast_c(plot=False): image, axis=0) - ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) ds = ds.map(input_columns=["image"], operations=[C.Decode(), C.Resize((224, 224))]) @@ -159,17 +161,18 @@ def test_auto_contrast_one_channel_c(plot=False): logger.info("Test AutoContrast C Op With One Channel Images") # AutoContrast Images - ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) ds = ds.map(input_columns=["image"], operations=[C.Decode(), C.Resize((224, 224))]) python_op = F.AutoContrast() c_op = C.AutoContrast() # not using F.ToTensor() since it converts to floats - transforms_op = F.ComposeOp([lambda img: (np.array(img)[:, :, 0]).astype(np.uint8), - F.ToPIL(), - python_op, - np.array])() + transforms_op = mindspore.dataset.transforms.py_transforms.Compose( + [lambda img: (np.array(img)[:, :, 0]).astype(np.uint8), + F.ToPIL(), + python_op, + np.array]) ds_auto_contrast_py = ds.map(input_columns="image", operations=transforms_op) @@ -184,7 +187,7 @@ def test_auto_contrast_one_channel_c(plot=False): image, axis=0) - ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) ds = ds.map(input_columns=["image"], operations=[C.Decode(), C.Resize((224, 224)), @@ -248,7 +251,7 @@ def test_auto_contrast_invalid_ignore_param_c(): """ logger.info("Test AutoContrast C Op with invalid ignore parameter") try: - ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) ds = ds.map(input_columns=["image"], operations=[C.Decode(), C.Resize((224, 224)), @@ -260,7 +263,7 @@ def test_auto_contrast_invalid_ignore_param_c(): logger.info("Got an exception in DE: {}".format(str(error))) assert "Argument ignore with value 255.5 is not of type" in str(error) try: - ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) ds = ds.map(input_columns=["image"], operations=[C.Decode(), C.Resize((224, 224)), @@ -279,7 +282,7 @@ def test_auto_contrast_invalid_cutoff_param_c(): """ logger.info("Test AutoContrast C Op with invalid cutoff parameter") try: - ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) ds = ds.map(input_columns=["image"], operations=[C.Decode(), C.Resize((224, 224)), @@ -291,7 +294,7 @@ def test_auto_contrast_invalid_cutoff_param_c(): logger.info("Got an exception in DE: {}".format(str(error))) assert "Input cutoff is not within the required interval of (0 to 100)." in str(error) try: - ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) ds = ds.map(input_columns=["image"], operations=[C.Decode(), C.Resize((224, 224)), @@ -310,22 +313,22 @@ def test_auto_contrast_invalid_ignore_param_py(): """ logger.info("Test AutoContrast python Op with invalid ignore parameter") try: - ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) ds = ds.map(input_columns=["image"], - operations=[F.ComposeOp([F.Decode(), - F.Resize((224, 224)), - F.AutoContrast(ignore=255.5), - F.ToTensor()])]) + operations=[mindspore.dataset.transforms.py_transforms.Compose([F.Decode(), + F.Resize((224, 224)), + F.AutoContrast(ignore=255.5), + F.ToTensor()])]) except TypeError as error: logger.info("Got an exception in DE: {}".format(str(error))) assert "Argument ignore with value 255.5 is not of type" in str(error) try: - ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) ds = ds.map(input_columns=["image"], - operations=[F.ComposeOp([F.Decode(), - F.Resize((224, 224)), - F.AutoContrast(ignore=(10, 100)), - F.ToTensor()])]) + operations=[mindspore.dataset.transforms.py_transforms.Compose([F.Decode(), + F.Resize((224, 224)), + F.AutoContrast(ignore=(10, 100)), + F.ToTensor()])]) except TypeError as error: logger.info("Got an exception in DE: {}".format(str(error))) assert "Argument ignore with value (10,100) is not of type" in str(error) @@ -337,22 +340,22 @@ def test_auto_contrast_invalid_cutoff_param_py(): """ logger.info("Test AutoContrast python Op with invalid cutoff parameter") try: - ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) ds = ds.map(input_columns=["image"], - operations=[F.ComposeOp([F.Decode(), - F.Resize((224, 224)), - F.AutoContrast(cutoff=-10.0), - F.ToTensor()])]) + operations=[mindspore.dataset.transforms.py_transforms.Compose([F.Decode(), + F.Resize((224, 224)), + F.AutoContrast(cutoff=-10.0), + F.ToTensor()])]) except ValueError as error: logger.info("Got an exception in DE: {}".format(str(error))) assert "Input cutoff is not within the required interval of (0 to 100)." in str(error) try: - ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) ds = ds.map(input_columns=["image"], - operations=[F.ComposeOp([F.Decode(), - F.Resize((224, 224)), - F.AutoContrast(cutoff=120.0), - F.ToTensor()])]) + operations=[mindspore.dataset.transforms.py_transforms.Compose([F.Decode(), + F.Resize((224, 224)), + F.AutoContrast(cutoff=120.0), + F.ToTensor()])]) except ValueError as error: logger.info("Got an exception in DE: {}".format(str(error))) assert "Input cutoff is not within the required interval of (0 to 100)." in str(error) diff --git a/tests/ut/python/dataset/test_batch.py b/tests/ut/python/dataset/test_batch.py index 1220d98344..9e5e0139e4 100644 --- a/tests/ut/python/dataset/test_batch.py +++ b/tests/ut/python/dataset/test_batch.py @@ -449,6 +449,22 @@ def test_batch_exception_13(): logger.info("Got an exception in DE: {}".format(str(e))) assert "shard_id" in str(e) + # test non-functional parameters + try: + data1 = data1.batch(batch_size, output_columns="3") + sum([1 for _ in data1]) + + except ValueError as e: + logger.info("Got an exception in DE: {}".format(str(e))) + assert "output_columns is currently not implemented." in str(e) + + try: + data1 = data1.batch(batch_size, column_order="3") + sum([1 for _ in data1]) + + except ValueError as e: + logger.info("Got an exception in DE: {}".format(str(e))) + assert "column_order is currently not implemented." in str(e) if __name__ == '__main__': test_batch_01() diff --git a/tests/ut/python/dataset/test_bounding_box_augment.py b/tests/ut/python/dataset/test_bounding_box_augment.py index f7ac962d91..9219f426c7 100644 --- a/tests/ut/python/dataset/test_bounding_box_augment.py +++ b/tests/ut/python/dataset/test_bounding_box_augment.py @@ -19,7 +19,7 @@ Testing the bounding box augment op in DE import numpy as np import mindspore.log as logger import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.c_transforms as c_vision +import mindspore.dataset.vision.c_transforms as c_vision from util import visualize_with_bounding_boxes, InvalidBBoxType, check_bad_bbox, \ config_get_set_seed, config_get_set_num_parallel_workers, save_and_check_md5 @@ -51,7 +51,7 @@ def test_bounding_box_augment_with_rotation_op(plot_vis=False): # map to apply ops dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"], output_columns=["image", "bbox"], - columns_order=["image", "bbox"], + column_order=["image", "bbox"], operations=[test_op]) filename = "bounding_box_augment_rotation_c_result.npz" @@ -90,7 +90,7 @@ def test_bounding_box_augment_with_crop_op(plot_vis=False): # map to apply ops dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"], output_columns=["image", "bbox"], - columns_order=["image", "bbox"], + column_order=["image", "bbox"], operations=[test_op]) filename = "bounding_box_augment_crop_c_result.npz" @@ -128,7 +128,7 @@ def test_bounding_box_augment_valid_ratio_c(plot_vis=False): # map to apply ops dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"], output_columns=["image", "bbox"], - columns_order=["image", "bbox"], + column_order=["image", "bbox"], operations=[test_op]) # Add column for "bbox" filename = "bounding_box_augment_valid_ratio_c_result.npz" @@ -165,7 +165,7 @@ def test_bounding_box_augment_op_coco_c(plot_vis=False): dataCoco2 = dataCoco2.map(input_columns=["image", "bbox"], output_columns=["image", "bbox"], - columns_order=["image", "bbox"], + column_order=["image", "bbox"], operations=[test_op]) unaugSamp, augSamp = [], [] @@ -197,17 +197,17 @@ def test_bounding_box_augment_valid_edge_c(plot_vis=False): # Add column for "bbox" dataVoc1 = dataVoc1.map(input_columns=["image", "bbox"], output_columns=["image", "bbox"], - columns_order=["image", "bbox"], + column_order=["image", "bbox"], operations=lambda img, bbox: (img, np.array([[0, 0, img.shape[1], img.shape[0], 0, 0, 0]]).astype(np.float32))) dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"], output_columns=["image", "bbox"], - columns_order=["image", "bbox"], + column_order=["image", "bbox"], operations=lambda img, bbox: (img, np.array([[0, 0, img.shape[1], img.shape[0], 0, 0, 0]]).astype(np.float32))) dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"], output_columns=["image", "bbox"], - columns_order=["image", "bbox"], + column_order=["image", "bbox"], operations=[test_op]) filename = "bounding_box_augment_valid_edge_c_result.npz" save_and_check_md5(dataVoc2, filename, generate_golden=GENERATE_GOLDEN) @@ -240,7 +240,7 @@ def test_bounding_box_augment_invalid_ratio_c(): # map to apply ops dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"], output_columns=["image", "bbox"], - columns_order=["image", "bbox"], + column_order=["image", "bbox"], operations=[test_op]) # Add column for "bbox" except ValueError as error: logger.info("Got an exception in DE: {}".format(str(error))) diff --git a/tests/ut/python/dataset/test_cache_map.py b/tests/ut/python/dataset/test_cache_map.py index d8218629eb..ac449b11d6 100644 --- a/tests/ut/python/dataset/test_cache_map.py +++ b/tests/ut/python/dataset/test_cache_map.py @@ -18,7 +18,7 @@ Testing cache operator with mappable datasets import os import pytest import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.c_transforms as c_vision +import mindspore.dataset.vision.c_transforms as c_vision from mindspore import log as logger from util import save_and_check_md5 @@ -46,7 +46,7 @@ def test_cache_map_basic1(): some_cache = ds.DatasetCache(session_id=1, size=0, spilling=True) # This DATA_DIR only has 2 images in it - ds1 = ds.ImageFolderDatasetV2(dataset_dir=DATA_DIR, cache=some_cache) + ds1 = ds.ImageFolderDataset(dataset_dir=DATA_DIR, cache=some_cache) decode_op = c_vision.Decode() ds1 = ds1.map(input_columns=["image"], operations=decode_op) ds1 = ds1.repeat(4) @@ -75,7 +75,7 @@ def test_cache_map_basic2(): some_cache = ds.DatasetCache(session_id=1, size=0, spilling=True) # This DATA_DIR only has 2 images in it - ds1 = ds.ImageFolderDatasetV2(dataset_dir=DATA_DIR) + ds1 = ds.ImageFolderDataset(dataset_dir=DATA_DIR) decode_op = c_vision.Decode() ds1 = ds1.map(input_columns=["image"], operations=decode_op, cache=some_cache) ds1 = ds1.repeat(4) @@ -104,7 +104,7 @@ def test_cache_map_basic3(): some_cache = ds.DatasetCache(session_id=1, size=0, spilling=True) # This DATA_DIR only has 2 images in it - ds1 = ds.ImageFolderDatasetV2(dataset_dir=DATA_DIR) + ds1 = ds.ImageFolderDataset(dataset_dir=DATA_DIR) decode_op = c_vision.Decode() ds1 = ds1.repeat(4) ds1 = ds1.map(input_columns=["image"], operations=decode_op, cache=some_cache) @@ -128,7 +128,7 @@ def test_cache_map_basic4(): some_cache = ds.DatasetCache(session_id=1, size=0, spilling=True) # This DATA_DIR only has 2 images in it - ds1 = ds.ImageFolderDatasetV2(dataset_dir=DATA_DIR, cache=some_cache) + ds1 = ds.ImageFolderDataset(dataset_dir=DATA_DIR, cache=some_cache) decode_op = c_vision.Decode() ds1 = ds1.repeat(4) ds1 = ds1.map(input_columns=["image"], operations=decode_op) @@ -165,7 +165,7 @@ def test_cache_map_failure1(): some_cache = ds.DatasetCache(session_id=1, size=0, spilling=True) # This DATA_DIR only has 2 images in it - ds1 = ds.ImageFolderDatasetV2(dataset_dir=DATA_DIR, cache=some_cache) + ds1 = ds.ImageFolderDataset(dataset_dir=DATA_DIR, cache=some_cache) decode_op = c_vision.Decode() ds1 = ds1.map(input_columns=["image"], operations=decode_op, cache=some_cache) ds1 = ds1.repeat(4) diff --git a/tests/ut/python/dataset/test_cache_nomap.py b/tests/ut/python/dataset/test_cache_nomap.py index 74fb0939a5..6688737f43 100644 --- a/tests/ut/python/dataset/test_cache_nomap.py +++ b/tests/ut/python/dataset/test_cache_nomap.py @@ -19,7 +19,7 @@ import os import pytest import mindspore.common.dtype as mstype import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.c_transforms as c_vision +import mindspore.dataset.vision.c_transforms as c_vision from mindspore import log as logger DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] diff --git a/tests/ut/python/dataset/test_center_crop.py b/tests/ut/python/dataset/test_center_crop.py index 86961857a5..786af70d23 100644 --- a/tests/ut/python/dataset/test_center_crop.py +++ b/tests/ut/python/dataset/test_center_crop.py @@ -17,8 +17,9 @@ Testing CenterCrop op in DE """ import numpy as np import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.c_transforms as vision -import mindspore.dataset.transforms.vision.py_transforms as py_vision +import mindspore.dataset.transforms.py_transforms +import mindspore.dataset.vision.c_transforms as vision +import mindspore.dataset.vision.py_transforms as py_vision from mindspore import log as logger from util import diff_mse, visualize_list, save_and_check_md5 @@ -93,8 +94,8 @@ def test_center_crop_comp(height=375, width=375, plot=False): py_vision.CenterCrop([height, width]), py_vision.ToTensor() ] - transform = py_vision.ComposeOp(transforms) - data2 = data2.map(input_columns=["image"], operations=transform()) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) + data2 = data2.map(input_columns=["image"], operations=transform) image_c_cropped = [] image_py_cropped = [] @@ -123,9 +124,9 @@ def test_crop_grayscale(height=375, width=375): (lambda image: (image.transpose(1, 2, 0) * 255).astype(np.uint8)) ] - transform = py_vision.ComposeOp(transforms) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) - data1 = data1.map(input_columns=["image"], operations=transform()) + data1 = data1.map(input_columns=["image"], operations=transform) # If input is grayscale, the output dimensions should be single channel crop_gray = vision.CenterCrop([height, width]) diff --git a/tests/ut/python/dataset/test_concat.py b/tests/ut/python/dataset/test_concat.py index 8426663915..f66cac67ed 100644 --- a/tests/ut/python/dataset/test_concat.py +++ b/tests/ut/python/dataset/test_concat.py @@ -17,7 +17,8 @@ import numpy as np import mindspore.common.dtype as mstype import mindspore.dataset as ds import mindspore.dataset.transforms.c_transforms as C -import mindspore.dataset.transforms.vision.py_transforms as F +import mindspore.dataset.transforms.py_transforms +import mindspore.dataset.vision.py_transforms as F from mindspore import log as logger @@ -317,15 +318,15 @@ def test_concat_14(): DATA_DIR = "../data/dataset/testPK/data" DATA_DIR2 = "../data/dataset/testImageNetData/train/" - data1 = ds.ImageFolderDatasetV2(DATA_DIR, num_samples=3) - data2 = ds.ImageFolderDatasetV2(DATA_DIR2, num_samples=2) + data1 = ds.ImageFolderDataset(DATA_DIR, num_samples=3) + data2 = ds.ImageFolderDataset(DATA_DIR2, num_samples=2) - transforms1 = F.ComposeOp([F.Decode(), - F.Resize((224, 224)), - F.ToTensor()]) + transforms1 = mindspore.dataset.transforms.py_transforms.Compose([F.Decode(), + F.Resize((224, 224)), + F.ToTensor()]) - data1 = data1.map(input_columns=["image"], operations=transforms1()) - data2 = data2.map(input_columns=["image"], operations=transforms1()) + data1 = data1.map(input_columns=["image"], operations=transforms1) + data2 = data2.map(input_columns=["image"], operations=transforms1) data3 = data1 + data2 expected, output = [], [] @@ -351,7 +352,7 @@ def test_concat_15(): DATA_DIR = "../data/dataset/testPK/data" DATA_DIR2 = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] - data1 = ds.ImageFolderDatasetV2(DATA_DIR) + data1 = ds.ImageFolderDataset(DATA_DIR) data2 = ds.TFRecordDataset(DATA_DIR2, columns_list=["image"]) data1 = data1.project(["image"]) diff --git a/tests/ut/python/dataset/test_concatenate_op.py b/tests/ut/python/dataset/test_concatenate_op.py index d60cff06c5..b404a565c0 100644 --- a/tests/ut/python/dataset/test_concatenate_op.py +++ b/tests/ut/python/dataset/test_concatenate_op.py @@ -74,7 +74,7 @@ def test_concatenate_op_multi_input_string(): concatenate_op = data_trans.Concatenate(0, prepend=prepend_tensor, append=append_tensor) - data = data.map(input_columns=["col1", "col2"], columns_order=["out1"], output_columns=["out1"], + data = data.map(input_columns=["col1", "col2"], column_order=["out1"], output_columns=["out1"], operations=concatenate_op) expected = np.array(["dw", "df", "1", "2", "d", "3", "4", "e", "dwsdf", "df"], dtype='S') for data_row in data: @@ -89,7 +89,7 @@ def test_concatenate_op_multi_input_numeric(): concatenate_op = data_trans.Concatenate(0, prepend=prepend_tensor) - data = data.map(input_columns=["col1", "col2"], columns_order=["out1"], output_columns=["out1"], + data = data.map(input_columns=["col1", "col2"], column_order=["out1"], output_columns=["out1"], operations=concatenate_op) expected = np.array([3, 5, 1, 2, 3, 4]) for data_row in data: diff --git a/tests/ut/python/dataset/test_config.py b/tests/ut/python/dataset/test_config.py index 2bff816659..0677ea5ca3 100644 --- a/tests/ut/python/dataset/test_config.py +++ b/tests/ut/python/dataset/test_config.py @@ -21,8 +21,9 @@ import glob import numpy as np import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.c_transforms as c_vision -import mindspore.dataset.transforms.vision.py_transforms as py_vision +import mindspore.dataset.transforms.py_transforms +import mindspore.dataset.vision.c_transforms as c_vision +import mindspore.dataset.vision.py_transforms as py_vision from mindspore import log as logger from util import dataset_equal @@ -283,8 +284,8 @@ def test_deterministic_python_seed(): py_vision.RandomCrop([512, 512], [200, 200, 200, 200]), py_vision.ToTensor(), ] - transform = py_vision.ComposeOp(transforms) - data1 = data1.map(input_columns=["image"], operations=transform()) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) + data1 = data1.map(input_columns=["image"], operations=transform) data1_output = [] # config.set_seed() calls random.seed() for data_one in data1.create_dict_iterator(num_epochs=1): @@ -292,7 +293,7 @@ def test_deterministic_python_seed(): # Second dataset data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) - data2 = data2.map(input_columns=["image"], operations=transform()) + data2 = data2.map(input_columns=["image"], operations=transform) # config.set_seed() calls random.seed(), resets seed for next dataset iterator ds.config.set_seed(0) @@ -326,8 +327,8 @@ def test_deterministic_python_seed_multi_thread(): py_vision.RandomCrop([512, 512], [200, 200, 200, 200]), py_vision.ToTensor(), ] - transform = py_vision.ComposeOp(transforms) - data1 = data1.map(input_columns=["image"], operations=transform(), python_multiprocessing=True) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) + data1 = data1.map(input_columns=["image"], operations=transform, python_multiprocessing=True) data1_output = [] # config.set_seed() calls random.seed() for data_one in data1.create_dict_iterator(num_epochs=1): @@ -336,7 +337,7 @@ def test_deterministic_python_seed_multi_thread(): # Second dataset data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) # If seed is set up on constructor - data2 = data2.map(input_columns=["image"], operations=transform(), python_multiprocessing=True) + data2 = data2.map(input_columns=["image"], operations=transform, python_multiprocessing=True) # config.set_seed() calls random.seed() ds.config.set_seed(0) diff --git a/tests/ut/python/dataset/test_cut_out.py b/tests/ut/python/dataset/test_cut_out.py index 1e9cf4090d..61b849b08d 100644 --- a/tests/ut/python/dataset/test_cut_out.py +++ b/tests/ut/python/dataset/test_cut_out.py @@ -18,8 +18,9 @@ Testing CutOut op in DE import numpy as np import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.c_transforms as c -import mindspore.dataset.transforms.vision.py_transforms as f +import mindspore.dataset.transforms.py_transforms +import mindspore.dataset.vision.c_transforms as c +import mindspore.dataset.vision.py_transforms as f from mindspore import log as logger from util import visualize_image, visualize_list, diff_mse, save_and_check_md5, \ config_get_set_seed, config_get_set_num_parallel_workers @@ -43,8 +44,8 @@ def test_cut_out_op(plot=False): f.ToTensor(), f.RandomErasing(value='random') ] - transform_1 = f.ComposeOp(transforms_1) - data1 = data1.map(input_columns=["image"], operations=transform_1()) + transform_1 = mindspore.dataset.transforms.py_transforms.Compose(transforms_1) + data1 = data1.map(input_columns=["image"], operations=transform_1) # Second dataset data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) @@ -89,8 +90,8 @@ def test_cut_out_op_multicut(plot=False): f.Decode(), f.ToTensor(), ] - transform_1 = f.ComposeOp(transforms_1) - data1 = data1.map(input_columns=["image"], operations=transform_1()) + transform_1 = mindspore.dataset.transforms.py_transforms.Compose(transforms_1) + data1 = data1.map(input_columns=["image"], operations=transform_1) # Second dataset data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) @@ -144,8 +145,8 @@ def test_cut_out_md5(): f.ToTensor(), f.Cutout(100) ] - transform = f.ComposeOp(transforms) - data2 = data2.map(input_columns=["image"], operations=transform()) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) + data2 = data2.map(input_columns=["image"], operations=transform) # Compare with expected md5 from images filename1 = "cut_out_01_c_result.npz" @@ -172,8 +173,8 @@ def test_cut_out_comp(plot=False): f.ToTensor(), f.Cutout(200) ] - transform_1 = f.ComposeOp(transforms_1) - data1 = data1.map(input_columns=["image"], operations=transform_1()) + transform_1 = mindspore.dataset.transforms.py_transforms.Compose(transforms_1) + data1 = data1.map(input_columns=["image"], operations=transform_1) # Second dataset data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) diff --git a/tests/ut/python/dataset/test_cutmix_batch_op.py b/tests/ut/python/dataset/test_cutmix_batch_op.py index ee7599e339..35e1a4be61 100644 --- a/tests/ut/python/dataset/test_cutmix_batch_op.py +++ b/tests/ut/python/dataset/test_cutmix_batch_op.py @@ -18,9 +18,9 @@ Testing the CutMixBatch op in DE import numpy as np import pytest import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.c_transforms as vision +import mindspore.dataset.vision.c_transforms as vision import mindspore.dataset.transforms.c_transforms as data_trans -import mindspore.dataset.transforms.vision.utils as mode +import mindspore.dataset.vision.utils as mode from mindspore import log as logger from util import save_and_check_md5, diff_mse, visualize_list, config_get_set_seed, \ config_get_set_num_parallel_workers @@ -119,11 +119,11 @@ def test_cutmix_batch_success2(plot=False): def test_cutmix_batch_success3(plot=False): """ - Test CutMixBatch op with default values for alpha and prob on a batch of HWC images on ImageFolderDatasetV2 + Test CutMixBatch op with default values for alpha and prob on a batch of HWC images on ImageFolderDataset """ logger.info("test_cutmix_batch_success3") - ds_original = ds.ImageFolderDatasetV2(dataset_dir=DATA_DIR2, shuffle=False) + ds_original = ds.ImageFolderDataset(dataset_dir=DATA_DIR2, shuffle=False) decode_op = vision.Decode() ds_original = ds_original.map(input_columns=["image"], operations=[decode_op]) ds_original = ds_original.batch(4, pad_info={}, drop_remainder=True) @@ -136,7 +136,7 @@ def test_cutmix_batch_success3(plot=False): images_original = np.append(images_original, image, axis=0) # CutMix Images - data1 = ds.ImageFolderDatasetV2(dataset_dir=DATA_DIR2, shuffle=False) + data1 = ds.ImageFolderDataset(dataset_dir=DATA_DIR2, shuffle=False) decode_op = vision.Decode() data1 = data1.map(input_columns=["image"], operations=[decode_op]) diff --git a/tests/ut/python/dataset/test_dataset_numpy_slices.py b/tests/ut/python/dataset/test_dataset_numpy_slices.py index 60930b5a14..83fee94346 100644 --- a/tests/ut/python/dataset/test_dataset_numpy_slices.py +++ b/tests/ut/python/dataset/test_dataset_numpy_slices.py @@ -18,7 +18,7 @@ import numpy as np import pandas as pd import mindspore.dataset as de from mindspore import log as logger -import mindspore.dataset.transforms.vision.c_transforms as vision +import mindspore.dataset.vision.c_transforms as vision def test_numpy_slices_list_1(): diff --git a/tests/ut/python/dataset/test_datasets_celeba.py b/tests/ut/python/dataset/test_datasets_celeba.py index 409041b1a8..910c4c1db7 100644 --- a/tests/ut/python/dataset/test_datasets_celeba.py +++ b/tests/ut/python/dataset/test_datasets_celeba.py @@ -12,9 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.c_transforms as vision +import mindspore.dataset.vision.c_transforms as vision from mindspore import log as logger -from mindspore.dataset.transforms.vision import Inter +from mindspore.dataset.vision import Inter DATA_DIR = "../data/dataset/testCelebAData/" diff --git a/tests/ut/python/dataset/test_datasets_coco.py b/tests/ut/python/dataset/test_datasets_coco.py index deff626333..39609092bf 100644 --- a/tests/ut/python/dataset/test_datasets_coco.py +++ b/tests/ut/python/dataset/test_datasets_coco.py @@ -14,7 +14,7 @@ # ============================================================================== import numpy as np import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.c_transforms as vision +import mindspore.dataset.vision.c_transforms as vision DATA_DIR = "../data/dataset/testCOCO/train/" DATA_DIR_2 = "../data/dataset/testCOCO/train" diff --git a/tests/ut/python/dataset/test_datasets_generator.py b/tests/ut/python/dataset/test_datasets_generator.py index d78b55aa17..a48875973a 100644 --- a/tests/ut/python/dataset/test_datasets_generator.py +++ b/tests/ut/python/dataset/test_datasets_generator.py @@ -244,7 +244,7 @@ def test_generator_8(): data1 = data1.map(input_columns="col0", output_columns="out0", operations=(lambda x: x * 3), num_parallel_workers=2) data1 = data1.map(input_columns="col1", output_columns=["out1", "out2"], operations=(lambda x: (x * 7, x)), - num_parallel_workers=2, columns_order=["out0", "out1", "out2"]) + num_parallel_workers=2, column_order=["out0", "out1", "out2"]) data1 = data1.map(input_columns="out2", output_columns="out2", operations=(lambda x: x + 1), num_parallel_workers=2) @@ -299,7 +299,7 @@ def test_generator_10(): # apply dataset operations data1 = ds.GeneratorDataset(generator_mc(2048), ["col0", "col1"]) data1 = data1.map(input_columns="col1", output_columns=["out1", "out2"], operations=(lambda x: (x, x * 5)), - columns_order=['col0', 'out1', 'out2'], num_parallel_workers=2) + column_order=['col0', 'out1', 'out2'], num_parallel_workers=2) # Expected column order is |col0|out1|out2| i = 0 @@ -318,17 +318,17 @@ def test_generator_11(): Test map column order when len(input_columns) != len(output_columns). """ logger.info("Test map column order when len(input_columns) != len(output_columns), " - "and columns_order drops some columns.") + "and column_order drops some columns.") # apply dataset operations data1 = ds.GeneratorDataset(generator_mc(2048), ["col0", "col1"]) data1 = data1.map(input_columns="col1", output_columns=["out1", "out2"], operations=(lambda x: (x, x * 5)), - columns_order=['out1', 'out2'], num_parallel_workers=2) + column_order=['out1', 'out2'], num_parallel_workers=2) # Expected column order is |out1|out2| i = 0 for item in data1.create_tuple_iterator(num_epochs=1): - # len should be 2 because col0 is dropped (not included in columns_order) + # len should be 2 because col0 is dropped (not included in column_order) assert len(item) == 2 golden = np.array([[i, i + 1], [i + 2, i + 3]]) np.testing.assert_array_equal(item[0], golden) @@ -358,7 +358,7 @@ def test_generator_12(): i = i + 1 data1 = ds.GeneratorDataset(generator_mc(2048), ["col0", "col1"]) - data1 = data1.map(operations=(lambda x: (x * 5)), columns_order=["col1", "col0"], num_parallel_workers=2) + data1 = data1.map(operations=(lambda x: (x * 5)), column_order=["col1", "col0"], num_parallel_workers=2) # Expected column order is |col0|col1| i = 0 @@ -392,7 +392,7 @@ def test_generator_13(): i = i + 1 for item in data1.create_dict_iterator(num_epochs=1): # each data is a dictionary - # len should be 2 because col0 is dropped (not included in columns_order) + # len should be 2 because col0 is dropped (not included in column_order) assert len(item) == 2 golden = np.array([i * 5]) np.testing.assert_array_equal(item["out0"], golden) @@ -508,7 +508,7 @@ def test_generator_error_3(): for _ in data1: pass - assert "When (len(input_columns) != len(output_columns)), columns_order must be specified." in str(info.value) + assert "When (len(input_columns) != len(output_columns)), column_order must be specified." in str(info.value) def test_generator_error_4(): diff --git a/tests/ut/python/dataset/test_datasets_get_dataset_size.py b/tests/ut/python/dataset/test_datasets_get_dataset_size.py index d1bef0f964..0bc7d7ce03 100644 --- a/tests/ut/python/dataset/test_datasets_get_dataset_size.py +++ b/tests/ut/python/dataset/test_datasets_get_dataset_size.py @@ -27,16 +27,16 @@ CIFAR100_DATA_DIR = "../data/dataset/testCifar100Data" def test_imagenet_rawdata_dataset_size(): - ds_total = ds.ImageFolderDatasetV2(IMAGENET_RAWDATA_DIR) + ds_total = ds.ImageFolderDataset(IMAGENET_RAWDATA_DIR) assert ds_total.get_dataset_size() == 6 - ds_shard_1_0 = ds.ImageFolderDatasetV2(IMAGENET_RAWDATA_DIR, num_shards=1, shard_id=0) + ds_shard_1_0 = ds.ImageFolderDataset(IMAGENET_RAWDATA_DIR, num_shards=1, shard_id=0) assert ds_shard_1_0.get_dataset_size() == 6 - ds_shard_2_0 = ds.ImageFolderDatasetV2(IMAGENET_RAWDATA_DIR, num_shards=2, shard_id=0) + ds_shard_2_0 = ds.ImageFolderDataset(IMAGENET_RAWDATA_DIR, num_shards=2, shard_id=0) assert ds_shard_2_0.get_dataset_size() == 3 - ds_shard_3_0 = ds.ImageFolderDatasetV2(IMAGENET_RAWDATA_DIR, num_shards=3, shard_id=0) + ds_shard_3_0 = ds.ImageFolderDataset(IMAGENET_RAWDATA_DIR, num_shards=3, shard_id=0) assert ds_shard_3_0.get_dataset_size() == 2 diff --git a/tests/ut/python/dataset/test_datasets_imagefolder.py b/tests/ut/python/dataset/test_datasets_imagefolder.py index f1e0d160f0..fefa0f16d8 100644 --- a/tests/ut/python/dataset/test_datasets_imagefolder.py +++ b/tests/ut/python/dataset/test_datasets_imagefolder.py @@ -24,7 +24,7 @@ def test_imagefolder_basic(): repeat_count = 1 # apply dataset operations - data1 = ds.ImageFolderDatasetV2(DATA_DIR) + data1 = ds.ImageFolderDataset(DATA_DIR) data1 = data1.repeat(repeat_count) num_iter = 0 @@ -44,7 +44,7 @@ def test_imagefolder_numsamples(): repeat_count = 1 # apply dataset operations - data1 = ds.ImageFolderDatasetV2(DATA_DIR, num_samples=10, num_parallel_workers=2) + data1 = ds.ImageFolderDataset(DATA_DIR, num_samples=10, num_parallel_workers=2) data1 = data1.repeat(repeat_count) num_iter = 0 @@ -58,7 +58,7 @@ def test_imagefolder_numsamples(): assert num_iter == 10 random_sampler = ds.RandomSampler(num_samples=3, replacement=True) - data1 = ds.ImageFolderDatasetV2(DATA_DIR, num_parallel_workers=2, sampler=random_sampler) + data1 = ds.ImageFolderDataset(DATA_DIR, num_parallel_workers=2, sampler=random_sampler) num_iter = 0 for item in data1.create_dict_iterator(num_epochs=1): @@ -67,7 +67,7 @@ def test_imagefolder_numsamples(): assert num_iter == 3 random_sampler = ds.RandomSampler(num_samples=3, replacement=False) - data1 = ds.ImageFolderDatasetV2(DATA_DIR, num_parallel_workers=2, sampler=random_sampler) + data1 = ds.ImageFolderDataset(DATA_DIR, num_parallel_workers=2, sampler=random_sampler) num_iter = 0 for item in data1.create_dict_iterator(num_epochs=1): @@ -82,7 +82,7 @@ def test_imagefolder_numshards(): repeat_count = 1 # apply dataset operations - data1 = ds.ImageFolderDatasetV2(DATA_DIR, num_shards=4, shard_id=3) + data1 = ds.ImageFolderDataset(DATA_DIR, num_shards=4, shard_id=3) data1 = data1.repeat(repeat_count) num_iter = 0 @@ -102,7 +102,7 @@ def test_imagefolder_shardid(): repeat_count = 1 # apply dataset operations - data1 = ds.ImageFolderDatasetV2(DATA_DIR, num_shards=4, shard_id=1) + data1 = ds.ImageFolderDataset(DATA_DIR, num_shards=4, shard_id=1) data1 = data1.repeat(repeat_count) num_iter = 0 @@ -122,7 +122,7 @@ def test_imagefolder_noshuffle(): repeat_count = 1 # apply dataset operations - data1 = ds.ImageFolderDatasetV2(DATA_DIR, shuffle=False) + data1 = ds.ImageFolderDataset(DATA_DIR, shuffle=False) data1 = data1.repeat(repeat_count) num_iter = 0 @@ -142,7 +142,7 @@ def test_imagefolder_extrashuffle(): repeat_count = 2 # apply dataset operations - data1 = ds.ImageFolderDatasetV2(DATA_DIR, shuffle=True) + data1 = ds.ImageFolderDataset(DATA_DIR, shuffle=True) data1 = data1.shuffle(buffer_size=5) data1 = data1.repeat(repeat_count) @@ -164,7 +164,7 @@ def test_imagefolder_classindex(): # apply dataset operations class_index = {"class3": 333, "class1": 111} - data1 = ds.ImageFolderDatasetV2(DATA_DIR, class_indexing=class_index, shuffle=False) + data1 = ds.ImageFolderDataset(DATA_DIR, class_indexing=class_index, shuffle=False) data1 = data1.repeat(repeat_count) golden = [111, 111, 111, 111, 111, 111, 111, 111, 111, 111, 111, @@ -189,7 +189,7 @@ def test_imagefolder_negative_classindex(): # apply dataset operations class_index = {"class3": -333, "class1": 111} - data1 = ds.ImageFolderDatasetV2(DATA_DIR, class_indexing=class_index, shuffle=False) + data1 = ds.ImageFolderDataset(DATA_DIR, class_indexing=class_index, shuffle=False) data1 = data1.repeat(repeat_count) golden = [111, 111, 111, 111, 111, 111, 111, 111, 111, 111, 111, @@ -214,7 +214,7 @@ def test_imagefolder_extensions(): # apply dataset operations ext = [".jpg", ".JPEG"] - data1 = ds.ImageFolderDatasetV2(DATA_DIR, extensions=ext) + data1 = ds.ImageFolderDataset(DATA_DIR, extensions=ext) data1 = data1.repeat(repeat_count) num_iter = 0 @@ -235,7 +235,7 @@ def test_imagefolder_decode(): # apply dataset operations ext = [".jpg", ".JPEG"] - data1 = ds.ImageFolderDatasetV2(DATA_DIR, extensions=ext, decode=True) + data1 = ds.ImageFolderDataset(DATA_DIR, extensions=ext, decode=True) data1 = data1.repeat(repeat_count) num_iter = 0 @@ -262,7 +262,7 @@ def test_sequential_sampler(): # apply dataset operations sampler = ds.SequentialSampler() - data1 = ds.ImageFolderDatasetV2(DATA_DIR, sampler=sampler) + data1 = ds.ImageFolderDataset(DATA_DIR, sampler=sampler) data1 = data1.repeat(repeat_count) result = [] @@ -283,7 +283,7 @@ def test_random_sampler(): # apply dataset operations sampler = ds.RandomSampler() - data1 = ds.ImageFolderDatasetV2(DATA_DIR, sampler=sampler) + data1 = ds.ImageFolderDataset(DATA_DIR, sampler=sampler) data1 = data1.repeat(repeat_count) num_iter = 0 @@ -304,7 +304,7 @@ def test_distributed_sampler(): # apply dataset operations sampler = ds.DistributedSampler(10, 1) - data1 = ds.ImageFolderDatasetV2(DATA_DIR, sampler=sampler) + data1 = ds.ImageFolderDataset(DATA_DIR, sampler=sampler) data1 = data1.repeat(repeat_count) num_iter = 0 @@ -325,7 +325,7 @@ def test_pk_sampler(): # apply dataset operations sampler = ds.PKSampler(3) - data1 = ds.ImageFolderDatasetV2(DATA_DIR, sampler=sampler) + data1 = ds.ImageFolderDataset(DATA_DIR, sampler=sampler) data1 = data1.repeat(repeat_count) num_iter = 0 @@ -347,7 +347,7 @@ def test_subset_random_sampler(): # apply dataset operations indices = [0, 1, 2, 3, 4, 5, 12, 13, 14, 15, 16, 11] sampler = ds.SubsetRandomSampler(indices) - data1 = ds.ImageFolderDatasetV2(DATA_DIR, sampler=sampler) + data1 = ds.ImageFolderDataset(DATA_DIR, sampler=sampler) data1 = data1.repeat(repeat_count) num_iter = 0 @@ -369,7 +369,7 @@ def test_weighted_random_sampler(): # apply dataset operations weights = [1.0, 0.1, 0.02, 0.3, 0.4, 0.05, 1.2, 0.13, 0.14, 0.015, 0.16, 1.1] sampler = ds.WeightedRandomSampler(weights, 11) - data1 = ds.ImageFolderDatasetV2(DATA_DIR, sampler=sampler) + data1 = ds.ImageFolderDataset(DATA_DIR, sampler=sampler) data1 = data1.repeat(repeat_count) num_iter = 0 @@ -389,7 +389,7 @@ def test_imagefolder_rename(): repeat_count = 1 # apply dataset operations - data1 = ds.ImageFolderDatasetV2(DATA_DIR, num_samples=10) + data1 = ds.ImageFolderDataset(DATA_DIR, num_samples=10) data1 = data1.repeat(repeat_count) num_iter = 0 @@ -421,8 +421,8 @@ def test_imagefolder_zip(): repeat_count = 2 # apply dataset operations - data1 = ds.ImageFolderDatasetV2(DATA_DIR, num_samples=10) - data2 = ds.ImageFolderDatasetV2(DATA_DIR, num_samples=10) + data1 = ds.ImageFolderDataset(DATA_DIR, num_samples=10) + data2 = ds.ImageFolderDataset(DATA_DIR, num_samples=10) data1 = data1.repeat(repeat_count) # rename dataset2 for no conflict diff --git a/tests/ut/python/dataset/test_datasets_sharding.py b/tests/ut/python/dataset/test_datasets_sharding.py index db819885f0..9730b91fb0 100644 --- a/tests/ut/python/dataset/test_datasets_sharding.py +++ b/tests/ut/python/dataset/test_datasets_sharding.py @@ -20,9 +20,9 @@ def test_imagefolder_shardings(print_res=False): image_folder_dir = "../data/dataset/testPK/data" def sharding_config(num_shards, shard_id, num_samples, shuffle, class_index, repeat_cnt=1): - data1 = ds.ImageFolderDatasetV2(image_folder_dir, num_samples=num_samples, num_shards=num_shards, - shard_id=shard_id, - shuffle=shuffle, class_indexing=class_index, decode=True) + data1 = ds.ImageFolderDataset(image_folder_dir, num_samples=num_samples, num_shards=num_shards, + shard_id=shard_id, + shuffle=shuffle, class_indexing=class_index, decode=True) data1 = data1.repeat(repeat_cnt) res = [] for item in data1.create_dict_iterator(num_epochs=1): # each data is a dictionary diff --git a/tests/ut/python/dataset/test_datasets_voc.py b/tests/ut/python/dataset/test_datasets_voc.py index 107292eecc..7bc2aa88a8 100644 --- a/tests/ut/python/dataset/test_datasets_voc.py +++ b/tests/ut/python/dataset/test_datasets_voc.py @@ -13,7 +13,7 @@ # limitations under the License. # ============================================================================== import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.c_transforms as vision +import mindspore.dataset.vision.c_transforms as vision DATA_DIR = "../data/dataset/testVOC2012" IMAGE_SHAPE = [2268, 2268, 2268, 2268, 642, 607, 561, 596, 612, 2268] diff --git a/tests/ut/python/dataset/test_decode.py b/tests/ut/python/dataset/test_decode.py index 7b732d8263..6040446ffe 100644 --- a/tests/ut/python/dataset/test_decode.py +++ b/tests/ut/python/dataset/test_decode.py @@ -18,7 +18,7 @@ Testing Decode op in DE import cv2 import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.c_transforms as vision +import mindspore.dataset.vision.c_transforms as vision from mindspore import log as logger from util import diff_mse diff --git a/tests/ut/python/dataset/test_deviceop_cpu.py b/tests/ut/python/dataset/test_deviceop_cpu.py index b5f18665e0..5d0d320704 100644 --- a/tests/ut/python/dataset/test_deviceop_cpu.py +++ b/tests/ut/python/dataset/test_deviceop_cpu.py @@ -15,7 +15,7 @@ import time import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.c_transforms as vision +import mindspore.dataset.vision.c_transforms as vision from mindspore import log as logger DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] diff --git a/tests/ut/python/dataset/test_duplicate_op.py b/tests/ut/python/dataset/test_duplicate_op.py index 97a858bfa2..d0db32dba8 100644 --- a/tests/ut/python/dataset/test_duplicate_op.py +++ b/tests/ut/python/dataset/test_duplicate_op.py @@ -24,7 +24,7 @@ import mindspore.dataset.transforms.c_transforms as ops def compare(array): data = ds.NumpySlicesDataset([array], column_names="x") array = np.array(array) - data = data.map(input_columns=["x"], output_columns=["x", "y"], columns_order=["x", "y"], + data = data.map(input_columns=["x"], output_columns=["x", "y"], column_order=["x", "y"], operations=ops.Duplicate()) for d in data.create_dict_iterator(num_epochs=1): np.testing.assert_array_equal(array, d["x"]) diff --git a/tests/ut/python/dataset/test_epoch_ctrl.py b/tests/ut/python/dataset/test_epoch_ctrl.py index 17fc9b78f2..f9747ab0e0 100644 --- a/tests/ut/python/dataset/test_epoch_ctrl.py +++ b/tests/ut/python/dataset/test_epoch_ctrl.py @@ -21,7 +21,7 @@ import numpy as np import pytest import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.c_transforms as vision +import mindspore.dataset.vision.c_transforms as vision from mindspore import log as logger DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] diff --git a/tests/ut/python/dataset/test_equalize.py b/tests/ut/python/dataset/test_equalize.py index bdc8375b67..7d4b2590d5 100644 --- a/tests/ut/python/dataset/test_equalize.py +++ b/tests/ut/python/dataset/test_equalize.py @@ -18,8 +18,9 @@ Testing Equalize op in DE import numpy as np import mindspore.dataset.engine as de -import mindspore.dataset.transforms.vision.c_transforms as C -import mindspore.dataset.transforms.vision.py_transforms as F +import mindspore.dataset.transforms.py_transforms +import mindspore.dataset.vision.c_transforms as C +import mindspore.dataset.vision.py_transforms as F from mindspore import log as logger from util import visualize_list, visualize_one_channel_dataset, diff_mse, save_and_check_md5 @@ -36,14 +37,14 @@ def test_equalize_py(plot=False): logger.info("Test Equalize") # Original Images - ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) - transforms_original = F.ComposeOp([F.Decode(), - F.Resize((224, 224)), - F.ToTensor()]) + transforms_original = mindspore.dataset.transforms.py_transforms.Compose([F.Decode(), + F.Resize((224, 224)), + F.ToTensor()]) ds_original = ds.map(input_columns="image", - operations=transforms_original()) + operations=transforms_original) ds_original = ds_original.batch(512) @@ -56,15 +57,15 @@ def test_equalize_py(plot=False): axis=0) # Color Equalized Images - ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) - transforms_equalize = F.ComposeOp([F.Decode(), - F.Resize((224, 224)), - F.Equalize(), - F.ToTensor()]) + transforms_equalize = mindspore.dataset.transforms.py_transforms.Compose([F.Decode(), + F.Resize((224, 224)), + F.Equalize(), + F.ToTensor()]) ds_equalize = ds.map(input_columns="image", - operations=transforms_equalize()) + operations=transforms_equalize) ds_equalize = ds_equalize.batch(512) @@ -93,7 +94,7 @@ def test_equalize_c(plot=False): logger.info("Test Equalize cpp op") # Original Images - ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) transforms_original = [C.Decode(), C.Resize(size=[224, 224])] @@ -111,7 +112,7 @@ def test_equalize_c(plot=False): axis=0) # Equalize Images - ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) transform_equalize = [C.Decode(), C.Resize(size=[224, 224]), C.Equalize()] @@ -145,7 +146,7 @@ def test_equalize_py_c(plot=False): logger.info("Test Equalize cpp and python op") # equalize Images in cpp - ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) ds = ds.map(input_columns=["image"], operations=[C.Decode(), C.Resize((224, 224))]) @@ -163,17 +164,17 @@ def test_equalize_py_c(plot=False): axis=0) # Equalize images in python - ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) ds = ds.map(input_columns=["image"], operations=[C.Decode(), C.Resize((224, 224))]) - transforms_p_equalize = F.ComposeOp([lambda img: img.astype(np.uint8), - F.ToPIL(), - F.Equalize(), - np.array]) + transforms_p_equalize = mindspore.dataset.transforms.py_transforms.Compose([lambda img: img.astype(np.uint8), + F.ToPIL(), + F.Equalize(), + np.array]) ds_p_equalize = ds.map(input_columns="image", - operations=transforms_p_equalize()) + operations=transforms_p_equalize) ds_p_equalize = ds_p_equalize.batch(512) @@ -204,7 +205,7 @@ def test_equalize_one_channel(): c_op = C.Equalize() try: - ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) ds = ds.map(input_columns=["image"], operations=[C.Decode(), C.Resize((224, 224)), @@ -253,12 +254,12 @@ def test_equalize_md5_py(): logger.info("Test Equalize") # First dataset - data1 = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) - transforms = F.ComposeOp([F.Decode(), - F.Equalize(), - F.ToTensor()]) + data1 = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) + transforms = mindspore.dataset.transforms.py_transforms.Compose([F.Decode(), + F.Equalize(), + F.ToTensor()]) - data1 = data1.map(input_columns="image", operations=transforms()) + data1 = data1.map(input_columns="image", operations=transforms) # Compare with expected md5 from images filename = "equalize_01_result.npz" save_and_check_md5(data1, filename, generate_golden=GENERATE_GOLDEN) @@ -271,7 +272,7 @@ def test_equalize_md5_c(): logger.info("Test Equalize cpp op with md5 check") # Generate dataset - ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) transforms_equalize = [C.Decode(), C.Resize(size=[224, 224]), diff --git a/tests/ut/python/dataset/test_exceptions.py b/tests/ut/python/dataset/test_exceptions.py index 0e0c2d1b42..7b89ad3b43 100644 --- a/tests/ut/python/dataset/test_exceptions.py +++ b/tests/ut/python/dataset/test_exceptions.py @@ -15,7 +15,7 @@ import pytest import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.c_transforms as vision +import mindspore.dataset.vision.c_transforms as vision from mindspore import log as logger DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] diff --git a/tests/ut/python/dataset/test_filterop.py b/tests/ut/python/dataset/test_filterop.py index f5514ded86..2a7381b9a0 100644 --- a/tests/ut/python/dataset/test_filterop.py +++ b/tests/ut/python/dataset/test_filterop.py @@ -16,7 +16,7 @@ import numpy as np import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.c_transforms as cde +import mindspore.dataset.vision.c_transforms as cde DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json" diff --git a/tests/ut/python/dataset/test_five_crop.py b/tests/ut/python/dataset/test_five_crop.py index 3db693c1c7..c119368540 100644 --- a/tests/ut/python/dataset/test_five_crop.py +++ b/tests/ut/python/dataset/test_five_crop.py @@ -18,7 +18,8 @@ import pytest import numpy as np import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.py_transforms as vision +import mindspore.dataset.transforms.py_transforms +import mindspore.dataset.vision.py_transforms as vision from mindspore import log as logger from util import visualize_list, save_and_check_md5 @@ -39,8 +40,8 @@ def test_five_crop_op(plot=False): vision.Decode(), vision.ToTensor(), ] - transform_1 = vision.ComposeOp(transforms_1) - data1 = data1.map(input_columns=["image"], operations=transform_1()) + transform_1 = mindspore.dataset.transforms.py_transforms.Compose(transforms_1) + data1 = data1.map(input_columns=["image"], operations=transform_1) # Second dataset data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) @@ -49,8 +50,8 @@ def test_five_crop_op(plot=False): vision.FiveCrop(200), lambda images: np.stack([vision.ToTensor()(image) for image in images]) # 4D stack of 5 images ] - transform_2 = vision.ComposeOp(transforms_2) - data2 = data2.map(input_columns=["image"], operations=transform_2()) + transform_2 = mindspore.dataset.transforms.py_transforms.Compose(transforms_2) + data2 = data2.map(input_columns=["image"], operations=transform_2) num_iter = 0 for item1, item2 in zip(data1.create_dict_iterator(num_epochs=1), data2.create_dict_iterator(num_epochs=1)): @@ -83,8 +84,8 @@ def test_five_crop_error_msg(): vision.FiveCrop(200), vision.ToTensor() ] - transform = vision.ComposeOp(transforms) - data = data.map(input_columns=["image"], operations=transform()) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) + data = data.map(input_columns=["image"], operations=transform) with pytest.raises(RuntimeError) as info: for _ in data: @@ -108,8 +109,8 @@ def test_five_crop_md5(): vision.FiveCrop(100), lambda images: np.stack([vision.ToTensor()(image) for image in images]) # 4D stack of 5 images ] - transform = vision.ComposeOp(transforms) - data = data.map(input_columns=["image"], operations=transform()) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) + data = data.map(input_columns=["image"], operations=transform) # Compare with expected md5 from images filename = "five_crop_01_result.npz" save_and_check_md5(data, filename, generate_golden=GENERATE_GOLDEN) diff --git a/tests/ut/python/dataset/test_flat_map.py b/tests/ut/python/dataset/test_flat_map.py index 07a4495eaa..e8161bbbea 100644 --- a/tests/ut/python/dataset/test_flat_map.py +++ b/tests/ut/python/dataset/test_flat_map.py @@ -27,7 +27,7 @@ def test_flat_map_1(): def flat_map_func(x): data_dir = x[0].item().decode('utf8') - d = ds.ImageFolderDatasetV2(data_dir) + d = ds.ImageFolderDataset(data_dir) return d data = ds.TextFileDataset(DATA_FILE) @@ -47,7 +47,7 @@ def test_flat_map_2(): def flat_map_func_1(x): data_dir = x[0].item().decode('utf8') - d = ds.ImageFolderDatasetV2(data_dir) + d = ds.ImageFolderDataset(data_dir) return d def flat_map_func_2(x): diff --git a/tests/ut/python/dataset/test_get_col_names.py b/tests/ut/python/dataset/test_get_col_names.py index b2b4e210ff..38fe93c755 100644 --- a/tests/ut/python/dataset/test_get_col_names.py +++ b/tests/ut/python/dataset/test_get_col_names.py @@ -15,7 +15,7 @@ import numpy as np import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.c_transforms as vision +import mindspore.dataset.vision.c_transforms as vision CELEBA_DIR = "../data/dataset/testCelebAData" CIFAR10_DIR = "../data/dataset/testCifar10Data" @@ -75,7 +75,7 @@ def test_get_column_name_generator(): def test_get_column_name_imagefolder(): - data = ds.ImageFolderDatasetV2(IMAGE_FOLDER_DIR) + data = ds.ImageFolderDataset(IMAGE_FOLDER_DIR) assert data.get_col_names() == ["image", "label"] @@ -105,7 +105,7 @@ def test_get_column_name_map(): assert data.get_col_names() == ["col1", "label"] data = ds.Cifar10Dataset(CIFAR10_DIR) data = data.map(input_columns=["image"], operations=center_crop_op, output_columns=["col1", "col2"], - columns_order=["col2", "col1"]) + column_order=["col2", "col1"]) assert data.get_col_names() == ["col2", "col1"] diff --git a/tests/ut/python/dataset/test_get_size.py b/tests/ut/python/dataset/test_get_size.py index 1dce312a32..b1f9bfff61 100644 --- a/tests/ut/python/dataset/test_get_size.py +++ b/tests/ut/python/dataset/test_get_size.py @@ -150,13 +150,13 @@ def test_manifest(): def test_imagefolder(): - data = ds.ImageFolderDatasetV2("../data/dataset/testPK/data/") + data = ds.ImageFolderDataset("../data/dataset/testPK/data/") assert data.get_dataset_size() == 44 assert data.num_classes() == 4 data = data.shuffle(100) assert data.num_classes() == 4 - data = ds.ImageFolderDatasetV2("../data/dataset/testPK/data/", num_samples=10) + data = ds.ImageFolderDataset("../data/dataset/testPK/data/", num_samples=10) assert data.get_dataset_size() == 10 assert data.num_classes() == 4 diff --git a/tests/ut/python/dataset/test_invert.py b/tests/ut/python/dataset/test_invert.py index 4f70c5a7ee..3ffefc5d9d 100644 --- a/tests/ut/python/dataset/test_invert.py +++ b/tests/ut/python/dataset/test_invert.py @@ -18,8 +18,9 @@ Testing Invert op in DE import numpy as np import mindspore.dataset.engine as de -import mindspore.dataset.transforms.vision.py_transforms as F -import mindspore.dataset.transforms.vision.c_transforms as C +import mindspore.dataset.transforms.py_transforms +import mindspore.dataset.vision.py_transforms as F +import mindspore.dataset.vision.c_transforms as C from mindspore import log as logger from util import visualize_list, save_and_check_md5, diff_mse @@ -35,14 +36,14 @@ def test_invert_py(plot=False): logger.info("Test Invert Python op") # Original Images - ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) - transforms_original = F.ComposeOp([F.Decode(), - F.Resize((224, 224)), - F.ToTensor()]) + transforms_original = mindspore.dataset.transforms.py_transforms.Compose([F.Decode(), + F.Resize((224, 224)), + F.ToTensor()]) ds_original = ds.map(input_columns="image", - operations=transforms_original()) + operations=transforms_original) ds_original = ds_original.batch(512) @@ -55,15 +56,15 @@ def test_invert_py(plot=False): axis=0) # Color Inverted Images - ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) - transforms_invert = F.ComposeOp([F.Decode(), - F.Resize((224, 224)), - F.Invert(), - F.ToTensor()]) + transforms_invert = mindspore.dataset.transforms.py_transforms.Compose([F.Decode(), + F.Resize((224, 224)), + F.Invert(), + F.ToTensor()]) ds_invert = ds.map(input_columns="image", - operations=transforms_invert()) + operations=transforms_invert) ds_invert = ds_invert.batch(512) @@ -92,7 +93,7 @@ def test_invert_c(plot=False): logger.info("Test Invert cpp op") # Original Images - ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) transforms_original = [C.Decode(), C.Resize(size=[224, 224])] @@ -110,7 +111,7 @@ def test_invert_c(plot=False): axis=0) # Invert Images - ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) transform_invert = [C.Decode(), C.Resize(size=[224, 224]), C.Invert()] @@ -144,7 +145,7 @@ def test_invert_py_c(plot=False): logger.info("Test Invert cpp and python op") # Invert Images in cpp - ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) ds = ds.map(input_columns=["image"], operations=[C.Decode(), C.Resize((224, 224))]) @@ -162,17 +163,17 @@ def test_invert_py_c(plot=False): axis=0) # invert images in python - ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) ds = ds.map(input_columns=["image"], operations=[C.Decode(), C.Resize((224, 224))]) - transforms_p_invert = F.ComposeOp([lambda img: img.astype(np.uint8), - F.ToPIL(), - F.Invert(), - np.array]) + transforms_p_invert = mindspore.dataset.transforms.py_transforms.Compose([lambda img: img.astype(np.uint8), + F.ToPIL(), + F.Invert(), + np.array]) ds_p_invert = ds.map(input_columns="image", - operations=transforms_p_invert()) + operations=transforms_p_invert) ds_p_invert = ds_p_invert.batch(512) @@ -203,7 +204,7 @@ def test_invert_one_channel(): c_op = C.Invert() try: - ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) ds = ds.map(input_columns=["image"], operations=[C.Decode(), C.Resize((224, 224)), @@ -224,13 +225,13 @@ def test_invert_md5_py(): logger.info("Test Invert python op with md5 check") # Generate dataset - ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) - transforms_invert = F.ComposeOp([F.Decode(), - F.Invert(), - F.ToTensor()]) + transforms_invert = mindspore.dataset.transforms.py_transforms.Compose([F.Decode(), + F.Invert(), + F.ToTensor()]) - data = ds.map(input_columns="image", operations=transforms_invert()) + data = ds.map(input_columns="image", operations=transforms_invert) # Compare with expected md5 from images filename = "invert_01_result_py.npz" save_and_check_md5(data, filename, generate_golden=GENERATE_GOLDEN) @@ -243,7 +244,7 @@ def test_invert_md5_c(): logger.info("Test Invert cpp op with md5 check") # Generate dataset - ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) transforms_invert = [C.Decode(), C.Resize(size=[224, 224]), diff --git a/tests/ut/python/dataset/test_linear_transformation.py b/tests/ut/python/dataset/test_linear_transformation.py index 6d1c7829a2..7a82d5df8a 100644 --- a/tests/ut/python/dataset/test_linear_transformation.py +++ b/tests/ut/python/dataset/test_linear_transformation.py @@ -17,7 +17,8 @@ Testing LinearTransformation op in DE """ import numpy as np import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.py_transforms as py_vision +import mindspore.dataset.transforms.py_transforms +import mindspore.dataset.vision.py_transforms as py_vision from mindspore import log as logger from util import diff_mse, visualize_list, save_and_check_md5 @@ -46,11 +47,11 @@ def test_linear_transformation_op(plot=False): py_vision.CenterCrop([height, weight]), py_vision.ToTensor() ] - transform = py_vision.ComposeOp(transforms) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) # First dataset data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) - data1 = data1.map(input_columns=["image"], operations=transform()) + data1 = data1.map(input_columns=["image"], operations=transform) # Note: if transformation matrix is diagonal matrix with all 1 in diagonal, # the output matrix in expected to be the same as the input matrix. data1 = data1.map(input_columns=["image"], @@ -58,7 +59,7 @@ def test_linear_transformation_op(plot=False): # Second dataset data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) - data2 = data2.map(input_columns=["image"], operations=transform()) + data2 = data2.map(input_columns=["image"], operations=transform) image_transformed = [] image = [] @@ -96,8 +97,8 @@ def test_linear_transformation_md5(): py_vision.ToTensor(), py_vision.LinearTransformation(transformation_matrix, mean_vector) ] - transform = py_vision.ComposeOp(transforms) - data1 = data1.map(input_columns=["image"], operations=transform()) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) + data1 = data1.map(input_columns=["image"], operations=transform) # Compare with expected md5 from images filename = "linear_transformation_01_result.npz" @@ -126,8 +127,8 @@ def test_linear_transformation_exception_01(): py_vision.ToTensor(), py_vision.LinearTransformation(None, mean_vector) ] - transform = py_vision.ComposeOp(transforms) - data1 = data1.map(input_columns=["image"], operations=transform()) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) + data1 = data1.map(input_columns=["image"], operations=transform) except TypeError as e: logger.info("Got an exception in DE: {}".format(str(e))) assert "Argument transformation_matrix with value None is not of type (,)" in str(e) @@ -155,8 +156,8 @@ def test_linear_transformation_exception_02(): py_vision.ToTensor(), py_vision.LinearTransformation(transformation_matrix, None) ] - transform = py_vision.ComposeOp(transforms) - data1 = data1.map(input_columns=["image"], operations=transform()) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) + data1 = data1.map(input_columns=["image"], operations=transform) except TypeError as e: logger.info("Got an exception in DE: {}".format(str(e))) assert "Argument mean_vector with value None is not of type (,)" in str(e) @@ -185,8 +186,8 @@ def test_linear_transformation_exception_03(): py_vision.ToTensor(), py_vision.LinearTransformation(transformation_matrix, mean_vector) ] - transform = py_vision.ComposeOp(transforms) - data1 = data1.map(input_columns=["image"], operations=transform()) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) + data1 = data1.map(input_columns=["image"], operations=transform) except ValueError as e: logger.info("Got an exception in DE: {}".format(str(e))) assert "square matrix" in str(e) @@ -215,8 +216,8 @@ def test_linear_transformation_exception_04(): py_vision.ToTensor(), py_vision.LinearTransformation(transformation_matrix, mean_vector) ] - transform = py_vision.ComposeOp(transforms) - data1 = data1.map(input_columns=["image"], operations=transform()) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) + data1 = data1.map(input_columns=["image"], operations=transform) except ValueError as e: logger.info("Got an exception in DE: {}".format(str(e))) assert "should match" in str(e) diff --git a/tests/ut/python/dataset/test_minddataset.py b/tests/ut/python/dataset/test_minddataset.py index 0aa56d9206..db1052adab 100644 --- a/tests/ut/python/dataset/test_minddataset.py +++ b/tests/ut/python/dataset/test_minddataset.py @@ -25,9 +25,9 @@ import pytest import numpy as np import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.c_transforms as vision +import mindspore.dataset.vision.c_transforms as vision from mindspore import log as logger -from mindspore.dataset.transforms.vision import Inter +from mindspore.dataset.vision import Inter from mindspore.mindrecord import FileWriter FILES_NUM = 4 diff --git a/tests/ut/python/dataset/test_mixup_label_smoothing.py b/tests/ut/python/dataset/test_mixup_label_smoothing.py index dbee6617c9..33b076b86a 100644 --- a/tests/ut/python/dataset/test_mixup_label_smoothing.py +++ b/tests/ut/python/dataset/test_mixup_label_smoothing.py @@ -17,8 +17,8 @@ import numpy as np import mindspore.dataset as ds import mindspore.dataset.transforms.c_transforms as c import mindspore.dataset.transforms.py_transforms as f -import mindspore.dataset.transforms.vision.c_transforms as c_vision -import mindspore.dataset.transforms.vision.py_transforms as py_vision +import mindspore.dataset.vision.c_transforms as c_vision +import mindspore.dataset.vision.py_transforms as py_vision from mindspore import log as logger DATA_DIR = "../data/dataset/testImageNetData/train" @@ -33,14 +33,13 @@ def test_one_hot_op(): # define map operations # ds = de.ImageFolderDataset(DATA_DIR, schema=SCHEMA_DIR) - dataset = ds.ImageFolderDatasetV2(DATA_DIR) + dataset = ds.ImageFolderDataset(DATA_DIR) num_classes = 2 epsilon_para = 0.1 - transforms = [f.OneHotOp(num_classes=num_classes, smoothing_rate=epsilon_para), - ] - transform_label = py_vision.ComposeOp(transforms) - dataset = dataset.map(input_columns=["label"], operations=transform_label()) + transforms = [f.OneHotOp(num_classes=num_classes, smoothing_rate=epsilon_para),] + transform_label = f.Compose(transforms) + dataset = dataset.map(input_columns=["label"], operations=transform_label) golden_label = np.ones(num_classes) * epsilon_para / num_classes golden_label[1] = 1 - epsilon_para / num_classes @@ -63,7 +62,7 @@ def test_mix_up_single(): resize_width = 224 # Create dataset and define map operations - ds1 = ds.ImageFolderDatasetV2(DATA_DIR_2) + ds1 = ds.ImageFolderDataset(DATA_DIR_2) num_classes = 10 decode_op = c_vision.Decode() @@ -112,7 +111,7 @@ def test_mix_up_multi(): resize_width = 224 # Create dataset and define map operations - ds1 = ds.ImageFolderDatasetV2(DATA_DIR_2) + ds1 = ds.ImageFolderDataset(DATA_DIR_2) num_classes = 3 decode_op = c_vision.Decode() diff --git a/tests/ut/python/dataset/test_mixup_op.py b/tests/ut/python/dataset/test_mixup_op.py index 381db7960f..ab7a2f8355 100644 --- a/tests/ut/python/dataset/test_mixup_op.py +++ b/tests/ut/python/dataset/test_mixup_op.py @@ -18,7 +18,7 @@ Testing the MixUpBatch op in DE import numpy as np import pytest import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.c_transforms as vision +import mindspore.dataset.vision.c_transforms as vision import mindspore.dataset.transforms.c_transforms as data_trans from mindspore import log as logger from util import save_and_check_md5, diff_mse, visualize_list, config_get_set_seed, \ @@ -74,12 +74,12 @@ def test_mixup_batch_success1(plot=False): def test_mixup_batch_success2(plot=False): """ - Test MixUpBatch op with specified alpha parameter on ImageFolderDatasetV2 + Test MixUpBatch op with specified alpha parameter on ImageFolderDataset """ logger.info("test_mixup_batch_success2") # Original Images - ds_original = ds.ImageFolderDatasetV2(dataset_dir=DATA_DIR2, shuffle=False) + ds_original = ds.ImageFolderDataset(dataset_dir=DATA_DIR2, shuffle=False) decode_op = vision.Decode() ds_original = ds_original.map(input_columns=["image"], operations=[decode_op]) ds_original = ds_original.batch(4, pad_info={}, drop_remainder=True) @@ -92,7 +92,7 @@ def test_mixup_batch_success2(plot=False): images_original = np.append(images_original, image, axis=0) # MixUp Images - data1 = ds.ImageFolderDatasetV2(dataset_dir=DATA_DIR2, shuffle=False) + data1 = ds.ImageFolderDataset(dataset_dir=DATA_DIR2, shuffle=False) decode_op = vision.Decode() data1 = data1.map(input_columns=["image"], operations=[decode_op]) diff --git a/tests/ut/python/dataset/test_normalizeOp.py b/tests/ut/python/dataset/test_normalizeOp.py index 7c69d7481c..9303a86c80 100644 --- a/tests/ut/python/dataset/test_normalizeOp.py +++ b/tests/ut/python/dataset/test_normalizeOp.py @@ -17,8 +17,9 @@ Testing Normalize op in DE """ import numpy as np import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.c_transforms as c_vision -import mindspore.dataset.transforms.vision.py_transforms as py_vision +import mindspore.dataset.transforms.py_transforms +import mindspore.dataset.vision.c_transforms as c_vision +import mindspore.dataset.vision.py_transforms as py_vision from mindspore import log as logger from util import diff_mse, save_and_check_md5, visualize_image @@ -59,10 +60,10 @@ def util_test_normalize(mean, std, op_type): py_vision.ToTensor(), py_vision.Normalize(mean, std) ] - transform = py_vision.ComposeOp(transforms) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) # Generate dataset data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) - data = data.map(input_columns=["image"], operations=transform()) + data = data.map(input_columns=["image"], operations=transform) else: raise ValueError("Wrong parameter value") return data @@ -78,10 +79,10 @@ def util_test_normalize_grayscale(num_output_channels, mean, std): py_vision.ToTensor(), py_vision.Normalize(mean, std) ] - transform = py_vision.ComposeOp(transforms) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) # Generate dataset data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) - data = data.map(input_columns=["image"], operations=transform()) + data = data.map(input_columns=["image"], operations=transform) return data @@ -130,17 +131,17 @@ def test_normalize_op_py(plot=False): py_vision.Decode(), py_vision.ToTensor() ] - transform = py_vision.ComposeOp(transforms) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) normalize_op = py_vision.Normalize(mean, std) # First dataset data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) - data1 = data1.map(input_columns=["image"], operations=transform()) + data1 = data1.map(input_columns=["image"], operations=transform) data1 = data1.map(input_columns=["image"], operations=normalize_op) # Second dataset data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) - data2 = data2.map(input_columns=["image"], operations=transform()) + data2 = data2.map(input_columns=["image"], operations=transform) num_iter = 0 for item1, item2 in zip(data1.create_dict_iterator(num_epochs=1), data2.create_dict_iterator(num_epochs=1)): diff --git a/tests/ut/python/dataset/test_onehot_op.py b/tests/ut/python/dataset/test_onehot_op.py index ac92ab7fe7..062256f505 100644 --- a/tests/ut/python/dataset/test_onehot_op.py +++ b/tests/ut/python/dataset/test_onehot_op.py @@ -19,7 +19,7 @@ import numpy as np import mindspore.dataset as ds import mindspore.dataset.transforms.c_transforms as data_trans -import mindspore.dataset.transforms.vision.c_transforms as c_vision +import mindspore.dataset.vision.c_transforms as c_vision from mindspore import log as logger from util import dataset_equal_with_function @@ -47,7 +47,7 @@ def test_one_hot(): # First dataset data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) one_hot_op = data_trans.OneHot(num_classes=depth) - data1 = data1.map(input_columns=["label"], operations=one_hot_op, columns_order=["label"]) + data1 = data1.map(input_columns=["label"], operations=one_hot_op, column_order=["label"]) # Second dataset data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["label"], shuffle=False) diff --git a/tests/ut/python/dataset/test_opt.py b/tests/ut/python/dataset/test_opt.py index f40458ebcb..f4dfe189ea 100644 --- a/tests/ut/python/dataset/test_opt.py +++ b/tests/ut/python/dataset/test_opt.py @@ -15,30 +15,36 @@ import numpy as np import mindspore.dataset as ds +from util import config_get_set_seed, config_get_set_num_parallel_workers # Generate 1d int numpy array from 0 - 63 def generator_1d(): - for i in range(64): + for i in range(4): yield (np.array([i]),) def test_case_0(): """ - Test 1D Generator + Test 1D Generator. + Test without explicit kwargs for input args. """ + original_seed = config_get_set_seed(55) + original_num_parallel_workers = config_get_set_num_parallel_workers(1) - # apply dataset operations + # apply dataset qoperations data1 = ds.GeneratorDataset(generator_1d, ["data"]) - data1 = data1.shuffle(2) - - data1 = data1.map(["data"], operations=(lambda x: x)) - + data1 = data1.map((lambda x: x), ["data"]) data1 = data1.batch(2) - for _ in data1.create_dict_iterator(num_epochs=1): # each data is a dictionary - pass + expected_data = np.array([[[1], [2]], [[3], [0]]]) + for i, data_row in enumerate(data1): + np.testing.assert_array_equal(data_row[0], expected_data[i]) + + # Restore configuration + ds.config.set_seed(original_seed) + ds.config.set_num_parallel_workers((original_num_parallel_workers)) if __name__ == "__main__": diff --git a/tests/ut/python/dataset/test_opt_pass.py b/tests/ut/python/dataset/test_opt_pass.py index 97f3bcb4c2..91e06ad3a2 100644 --- a/tests/ut/python/dataset/test_opt_pass.py +++ b/tests/ut/python/dataset/test_opt_pass.py @@ -27,7 +27,7 @@ def test_map_reorder0(): # Generator -> Map data0 = ds.GeneratorDataset(generator_mc, ["col0", "col1"]) - data0 = data0.map(input_columns="col0", output_columns="out", columns_order=["col1", "out"], + data0 = data0.map(input_columns="col0", output_columns="out", column_order=["col1", "out"], operations=(lambda x: x)) for item in data0.create_tuple_iterator(num_epochs=1): # each data is a dictionary @@ -43,11 +43,11 @@ def test_map_reorder1(): # Three map and zip data0 = ds.GeneratorDataset(generator_mc, ["a0", "a1", "a2"]) - data0 = data0.map(input_columns="a0", columns_order=["a2", "a1", "a0"], operations=(lambda x: x)) + data0 = data0.map(input_columns="a0", column_order=["a2", "a1", "a0"], operations=(lambda x: x)) data1 = ds.GeneratorDataset(generator_mc, ["b0", "b1", "b2"]) - data1 = data1.map(input_columns="b0", columns_order=["b1", "b2", "b0"], operations=(lambda x: x)) + data1 = data1.map(input_columns="b0", column_order=["b1", "b2", "b0"], operations=(lambda x: x)) data2 = ds.zip((data0, data1)) - data2 = data2.map(input_columns="a0", columns_order=["b2", "a2", "b1", "a1", "b0", "a0"], operations=(lambda x: x)) + data2 = data2.map(input_columns="a0", column_order=["b2", "a2", "b1", "a1", "b0", "a0"], operations=(lambda x: x)) for item in data2.create_tuple_iterator(num_epochs=1): assert item == [np.array(2), np.array(2), np.array(1), np.array(1), np.array(0), np.array(0)] diff --git a/tests/ut/python/dataset/test_pad.py b/tests/ut/python/dataset/test_pad.py index 9c81ad5390..3814b7e979 100644 --- a/tests/ut/python/dataset/test_pad.py +++ b/tests/ut/python/dataset/test_pad.py @@ -18,8 +18,9 @@ Testing Pad op in DE import numpy as np import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.c_transforms as c_vision -import mindspore.dataset.transforms.vision.py_transforms as py_vision +import mindspore.dataset.transforms.py_transforms +import mindspore.dataset.vision.c_transforms as c_vision +import mindspore.dataset.vision.py_transforms as py_vision from mindspore import log as logger from util import diff_mse, save_and_check_md5 @@ -51,9 +52,9 @@ def test_pad_op(): py_vision.Pad(100), py_vision.ToTensor(), ] - transform = py_vision.ComposeOp(transforms) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) - data2 = data2.map(input_columns=["image"], operations=transform()) + data2 = data2.map(input_columns=["image"], operations=transform) for item1, item2 in zip(data1.create_dict_iterator(num_epochs=1), data2.create_dict_iterator(num_epochs=1)): c_image = item1["image"] @@ -85,9 +86,9 @@ def test_pad_grayscale(): (lambda image: (image.transpose(1, 2, 0) * 255).astype(np.uint8)) ] - transform = py_vision.ComposeOp(transforms) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) - data1 = data1.map(input_columns=["image"], operations=transform()) + data1 = data1.map(input_columns=["image"], operations=transform) # if input is grayscale, the output dimensions should be single channel pad_gray = c_vision.Pad(100, fill_value=(20, 20, 20)) @@ -140,8 +141,8 @@ def test_pad_md5(): py_vision.Pad(150), py_vision.ToTensor(), ] - transform = py_vision.ComposeOp(pytrans) - data2 = data2.map(input_columns=["image"], operations=transform()) + transform = mindspore.dataset.transforms.py_transforms.Compose(pytrans) + data2 = data2.map(input_columns=["image"], operations=transform) # Compare with expected md5 from images filename1 = "pad_01_c_result.npz" save_and_check_md5(data1, filename1, generate_golden=GENERATE_GOLDEN) diff --git a/tests/ut/python/dataset/test_paddeddataset.py b/tests/ut/python/dataset/test_paddeddataset.py index 4886991821..370690543e 100644 --- a/tests/ut/python/dataset/test_paddeddataset.py +++ b/tests/ut/python/dataset/test_paddeddataset.py @@ -5,7 +5,7 @@ import numpy as np import pytest import mindspore.dataset as ds from mindspore.mindrecord import FileWriter -import mindspore.dataset.transforms.vision.c_transforms as V_C +import mindspore.dataset.vision.c_transforms as V_C from PIL import Image FILES_NUM = 4 @@ -204,7 +204,7 @@ def test_raise_error(): def test_imagefolder_padded(): DATA_DIR = "../data/dataset/testPK/data" - data = ds.ImageFolderDatasetV2(DATA_DIR) + data = ds.ImageFolderDataset(DATA_DIR) data1 = [{'image': np.zeros(1, np.uint8), 'label': np.array(0, np.int32)}, {'image': np.zeros(2, np.uint8), 'label': np.array(1, np.int32)}, @@ -230,7 +230,7 @@ def test_imagefolder_padded_with_decode(): count = 0 for shard_id in range(num_shards): DATA_DIR = "../data/dataset/testPK/data" - data = ds.ImageFolderDatasetV2(DATA_DIR) + data = ds.ImageFolderDataset(DATA_DIR) white_io = BytesIO() Image.new('RGB', (224, 224), (255, 255, 255)).save(white_io, 'JPEG') @@ -258,7 +258,7 @@ def test_imagefolder_padded_with_decode_and_get_dataset_size(): count = 0 for shard_id in range(num_shards): DATA_DIR = "../data/dataset/testPK/data" - data = ds.ImageFolderDatasetV2(DATA_DIR) + data = ds.ImageFolderDataset(DATA_DIR) white_io = BytesIO() Image.new('RGB', (224, 224), (255, 255, 255)).save(white_io, 'JPEG') diff --git a/tests/ut/python/dataset/test_profiling.py b/tests/ut/python/dataset/test_profiling.py index a4ee68e435..486322cd4a 100644 --- a/tests/ut/python/dataset/test_profiling.py +++ b/tests/ut/python/dataset/test_profiling.py @@ -66,7 +66,7 @@ def test_profiling_complex_pipeline(): source = [(np.array([x]),) for x in range(1024)] data1 = ds.GeneratorDataset(source, ["gen"]) - data1 = data1.map("gen", operations=[(lambda x: x + 1)]) + data1 = data1.map(operations=[(lambda x: x + 1)], input_columns=["gen"]) pattern = DATASET_ROOT + "/test.data" data2 = ds.TFRecordDataset(pattern, SCHEMA_FILE, shuffle=ds.Shuffle.FILES) diff --git a/tests/ut/python/dataset/test_pyfunc.py b/tests/ut/python/dataset/test_pyfunc.py index 0e8aa49dde..5924d9ce12 100644 --- a/tests/ut/python/dataset/test_pyfunc.py +++ b/tests/ut/python/dataset/test_pyfunc.py @@ -54,7 +54,7 @@ def test_case_1(): # apply dataset operations data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) data1 = data1.map(input_columns=col, output_columns=["out0", "out1"], operations=(lambda x: (x, x + x)), - columns_order=["out0", "out1"]) + column_order=["out0", "out1"]) i = 0 for item in data1.create_dict_iterator(num_epochs=1): # each data is a dictionary @@ -78,7 +78,7 @@ def test_case_2(): data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) data1 = data1.map(input_columns=col, output_columns="out", operations=(lambda x, y: x + y), - columns_order=["out"]) + column_order=["out"]) i = 0 for item in data1.create_dict_iterator(num_epochs=1): # each data is a dictionary @@ -100,7 +100,7 @@ def test_case_3(): data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) data1 = data1.map(input_columns=col, output_columns=["out0", "out1", "out2"], - operations=(lambda x, y: (x, x + y, x + y + 1)), columns_order=["out0", "out1", "out2"]) + operations=(lambda x, y: (x, x + y, x + y + 1)), column_order=["out0", "out1", "out2"]) i = 0 for item in data1.create_dict_iterator(num_epochs=1): # each data is a dictionary @@ -126,7 +126,7 @@ def test_case_4(): data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) data1 = data1.map(input_columns=col, output_columns=["out0", "out1", "out2"], num_parallel_workers=4, - operations=(lambda x, y: (x, x + y, x + y + 1)), columns_order=["out0", "out1", "out2"]) + operations=(lambda x, y: (x, x + y, x + y + 1)), column_order=["out0", "out1", "out2"]) i = 0 for item in data1.create_dict_iterator(num_epochs=1): # each data is a dictionary @@ -166,7 +166,7 @@ def test_case_6(): """ Test PyFunc """ - logger.info("Test PyFunc ComposeOp : (lambda x : x + x), (lambda x : x + x)") + logger.info("Test PyFunc Compose : (lambda x : x + x), (lambda x : x + x)") # apply dataset operations data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) @@ -214,7 +214,7 @@ def test_case_8(): data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) data1 = data1.map(input_columns=col, output_columns=["out0", "out1", "out2"], num_parallel_workers=4, - operations=(lambda x, y: (x, x + y, x + y + 1)), columns_order=["out0", "out1", "out2"], + operations=(lambda x, y: (x, x + y, x + y + 1)), column_order=["out0", "out1", "out2"], python_multiprocessing=True) i = 0 diff --git a/tests/ut/python/dataset/test_random_affine.py b/tests/ut/python/dataset/test_random_affine.py index c99fa35e71..60cf7d0889 100644 --- a/tests/ut/python/dataset/test_random_affine.py +++ b/tests/ut/python/dataset/test_random_affine.py @@ -17,8 +17,9 @@ Testing RandomAffine op in DE """ import numpy as np import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.py_transforms as py_vision -import mindspore.dataset.transforms.vision.c_transforms as c_vision +import mindspore.dataset.transforms.py_transforms +import mindspore.dataset.vision.py_transforms as py_vision +import mindspore.dataset.vision.c_transforms as c_vision from mindspore import log as logger from util import visualize_list, save_and_check_md5, \ config_get_set_seed, config_get_set_num_parallel_workers @@ -41,20 +42,20 @@ def test_random_affine_op(plot=False): py_vision.RandomAffine(degrees=15, translate=(0.1, 0.1), scale=(0.9, 1.1)), py_vision.ToTensor() ] - transform1 = py_vision.ComposeOp(transforms1) + transform1 = mindspore.dataset.transforms.py_transforms.Compose(transforms1) transforms2 = [ py_vision.Decode(), py_vision.ToTensor() ] - transform2 = py_vision.ComposeOp(transforms2) + transform2 = mindspore.dataset.transforms.py_transforms.Compose(transforms2) # First dataset data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) - data1 = data1.map(input_columns=["image"], operations=transform1()) + data1 = data1.map(input_columns=["image"], operations=transform1) # Second dataset data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) - data2 = data2.map(input_columns=["image"], operations=transform2()) + data2 = data2.map(input_columns=["image"], operations=transform2) image_affine = [] image_original = [] @@ -114,11 +115,11 @@ def test_random_affine_md5(): scale=(0.9, 1.1), shear=(-10, 10, -5, 5)), py_vision.ToTensor() ] - transform = py_vision.ComposeOp(transforms) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) # Generate dataset data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) - data = data.map(input_columns=["image"], operations=transform()) + data = data.map(input_columns=["image"], operations=transform) # check results with md5 comparison filename = "random_affine_01_result.npz" @@ -189,9 +190,9 @@ def test_random_affine_py_exception_non_pil_images(): logger.info("test_random_affine_exception_negative_degrees") dataset = ds.MnistDataset(MNIST_DATA_DIR, num_parallel_workers=3) try: - transform = py_vision.ComposeOp([py_vision.ToTensor(), - py_vision.RandomAffine(degrees=(15, 15))]) - dataset = dataset.map(input_columns=["image"], operations=transform(), num_parallel_workers=3, + transform = mindspore.dataset.transforms.py_transforms.Compose([py_vision.ToTensor(), + py_vision.RandomAffine(degrees=(15, 15))]) + dataset = dataset.map(input_columns=["image"], operations=transform, num_parallel_workers=3, python_multiprocessing=True) for _ in dataset.create_dict_iterator(num_epochs=1): break diff --git a/tests/ut/python/dataset/test_random_apply.py b/tests/ut/python/dataset/test_random_apply.py index 49d828386f..5fca4cd7ed 100644 --- a/tests/ut/python/dataset/test_random_apply.py +++ b/tests/ut/python/dataset/test_random_apply.py @@ -17,7 +17,8 @@ Testing RandomApply op in DE """ import numpy as np import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.py_transforms as py_vision +import mindspore.dataset.transforms.py_transforms +import mindspore.dataset.vision.py_transforms as py_vision from mindspore import log as logger from util import visualize_list, config_get_set_seed, \ config_get_set_num_parallel_workers, save_and_check_md5 @@ -40,20 +41,20 @@ def test_random_apply_op(plot=False): py_vision.RandomApply(transforms_list, prob=0.6), py_vision.ToTensor() ] - transform1 = py_vision.ComposeOp(transforms1) + transform1 = mindspore.dataset.transforms.py_transforms.Compose(transforms1) transforms2 = [ py_vision.Decode(), py_vision.ToTensor() ] - transform2 = py_vision.ComposeOp(transforms2) + transform2 = mindspore.dataset.transforms.py_transforms.Compose(transforms2) # First dataset data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) - data1 = data1.map(input_columns=["image"], operations=transform1()) + data1 = data1.map(input_columns=["image"], operations=transform1) # Second dataset data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) - data2 = data2.map(input_columns=["image"], operations=transform2()) + data2 = data2.map(input_columns=["image"], operations=transform2) image_apply = [] image_original = [] @@ -81,11 +82,11 @@ def test_random_apply_md5(): py_vision.RandomApply(transforms_list), py_vision.ToTensor() ] - transform = py_vision.ComposeOp(transforms) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) # Generate dataset data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) - data = data.map(input_columns=["image"], operations=transform()) + data = data.map(input_columns=["image"], operations=transform) # check results with md5 comparison filename = "random_apply_01_result.npz" @@ -113,10 +114,10 @@ def test_random_apply_exception_random_crop_badinput(): py_vision.RandomApply(transforms_list, prob=0.6), py_vision.ToTensor() ] - transform = py_vision.ComposeOp(transforms) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) # Generate dataset data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) - data = data.map(input_columns=["image"], operations=transform()) + data = data.map(input_columns=["image"], operations=transform) try: _ = data.create_dict_iterator(num_epochs=1).get_next() except RuntimeError as e: diff --git a/tests/ut/python/dataset/test_random_choice.py b/tests/ut/python/dataset/test_random_choice.py index df632feb53..be7d06dff6 100644 --- a/tests/ut/python/dataset/test_random_choice.py +++ b/tests/ut/python/dataset/test_random_choice.py @@ -17,7 +17,8 @@ Testing RandomChoice op in DE """ import numpy as np import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.py_transforms as py_vision +import mindspore.dataset.transforms.py_transforms +import mindspore.dataset.vision.py_transforms as py_vision from mindspore import log as logger from util import visualize_list, diff_mse @@ -37,20 +38,20 @@ def test_random_choice_op(plot=False): py_vision.RandomChoice(transforms_list), py_vision.ToTensor() ] - transform1 = py_vision.ComposeOp(transforms1) + transform1 = mindspore.dataset.transforms.py_transforms.Compose(transforms1) transforms2 = [ py_vision.Decode(), py_vision.ToTensor() ] - transform2 = py_vision.ComposeOp(transforms2) + transform2 = mindspore.dataset.transforms.py_transforms.Compose(transforms2) # First dataset data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) - data1 = data1.map(input_columns=["image"], operations=transform1()) + data1 = data1.map(input_columns=["image"], operations=transform1) # Second dataset data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) - data2 = data2.map(input_columns=["image"], operations=transform2()) + data2 = data2.map(input_columns=["image"], operations=transform2) image_choice = [] image_original = [] @@ -75,21 +76,21 @@ def test_random_choice_comp(plot=False): py_vision.RandomChoice(transforms_list), py_vision.ToTensor() ] - transform1 = py_vision.ComposeOp(transforms1) + transform1 = mindspore.dataset.transforms.py_transforms.Compose(transforms1) transforms2 = [ py_vision.Decode(), py_vision.CenterCrop(64), py_vision.ToTensor() ] - transform2 = py_vision.ComposeOp(transforms2) + transform2 = mindspore.dataset.transforms.py_transforms.Compose(transforms2) # First dataset data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) - data1 = data1.map(input_columns=["image"], operations=transform1()) + data1 = data1.map(input_columns=["image"], operations=transform1) # Second dataset data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) - data2 = data2.map(input_columns=["image"], operations=transform2()) + data2 = data2.map(input_columns=["image"], operations=transform2) image_choice = [] image_original = [] @@ -119,10 +120,10 @@ def test_random_choice_exception_random_crop_badinput(): py_vision.RandomChoice(transforms_list), py_vision.ToTensor() ] - transform = py_vision.ComposeOp(transforms) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) # Generate dataset data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) - data = data.map(input_columns=["image"], operations=transform()) + data = data.map(input_columns=["image"], operations=transform) try: _ = data.create_dict_iterator(num_epochs=1).get_next() except RuntimeError as e: diff --git a/tests/ut/python/dataset/test_random_color.py b/tests/ut/python/dataset/test_random_color.py index 98b45d6050..f6e08c34c5 100644 --- a/tests/ut/python/dataset/test_random_color.py +++ b/tests/ut/python/dataset/test_random_color.py @@ -20,8 +20,9 @@ import pytest import mindspore.dataset as ds import mindspore.dataset.engine as de -import mindspore.dataset.transforms.vision.c_transforms as vision -import mindspore.dataset.transforms.vision.py_transforms as F +import mindspore.dataset.transforms.py_transforms +import mindspore.dataset.vision.c_transforms as vision +import mindspore.dataset.vision.py_transforms as F from mindspore import log as logger from util import visualize_list, diff_mse, save_and_check_md5, \ config_get_set_seed, config_get_set_num_parallel_workers @@ -43,14 +44,14 @@ def test_random_color_py(degrees=(0.1, 1.9), plot=False): logger.info("Test RandomColor") # Original Images - data = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + data = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) - transforms_original = F.ComposeOp([F.Decode(), - F.Resize((224, 224)), - F.ToTensor()]) + transforms_original = mindspore.dataset.transforms.py_transforms.Compose([F.Decode(), + F.Resize((224, 224)), + F.ToTensor()]) ds_original = data.map(input_columns="image", - operations=transforms_original()) + operations=transforms_original) ds_original = ds_original.batch(512) @@ -63,15 +64,15 @@ def test_random_color_py(degrees=(0.1, 1.9), plot=False): axis=0) # Random Color Adjusted Images - data = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + data = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) - transforms_random_color = F.ComposeOp([F.Decode(), - F.Resize((224, 224)), - F.RandomColor(degrees=degrees), - F.ToTensor()]) + transforms_random_color = mindspore.dataset.transforms.py_transforms.Compose([F.Decode(), + F.Resize((224, 224)), + F.RandomColor(degrees=degrees), + F.ToTensor()]) ds_random_color = data.map(input_columns="image", - operations=transforms_random_color()) + operations=transforms_random_color) ds_random_color = ds_random_color.batch(512) @@ -146,13 +147,13 @@ def test_random_color_py_md5(): original_num_parallel_workers = config_get_set_num_parallel_workers(1) # Generate dataset - data = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + data = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) - transforms = F.ComposeOp([F.Decode(), - F.RandomColor((2.0, 2.5)), - F.ToTensor()]) + transforms = mindspore.dataset.transforms.py_transforms.Compose([F.Decode(), + F.RandomColor((2.0, 2.5)), + F.ToTensor()]) - data = data.map(input_columns="image", operations=transforms()) + data = data.map(input_columns="image", operations=transforms) # Compare with expected md5 from images filename = "random_color_01_result.npz" save_and_check_md5(data, filename, generate_golden=GENERATE_GOLDEN) @@ -183,12 +184,12 @@ def test_compare_random_color_op(degrees=None, plot=False): c_op = vision.RandomColor(degrees) p_op = F.RandomColor(degrees) - transforms_random_color_py = F.ComposeOp([lambda img: img.astype(np.uint8), F.ToPIL(), - p_op, np.array]) + transforms_random_color_py = mindspore.dataset.transforms.py_transforms.Compose([lambda img: img.astype(np.uint8), F.ToPIL(), + p_op, np.array]) data1 = data1.map(input_columns=["image"], operations=[vision.Decode(), c_op]) data2 = data2.map(input_columns=["image"], operations=[vision.Decode()]) - data2 = data2.map(input_columns=["image"], operations=transforms_random_color_py()) + data2 = data2.map(input_columns=["image"], operations=transforms_random_color_py) image_random_color_op = [] image = [] @@ -249,7 +250,8 @@ if __name__ == "__main__": test_random_color_c() test_random_color_c(plot=True) - test_random_color_c(degrees=(2.0, 2.5), plot=True, run_golden=False) # Test with degree values that show more obvious transformation + test_random_color_c(degrees=(2.0, 2.5), plot=True, + run_golden=False) # Test with degree values that show more obvious transformation test_random_color_c(degrees=(0.1, 0.1), plot=True, run_golden=False) test_compare_random_color_op(plot=True) test_random_color_c_errors() diff --git a/tests/ut/python/dataset/test_random_color_adjust.py b/tests/ut/python/dataset/test_random_color_adjust.py index 3dad5549f5..6298bfdec7 100644 --- a/tests/ut/python/dataset/test_random_color_adjust.py +++ b/tests/ut/python/dataset/test_random_color_adjust.py @@ -19,8 +19,9 @@ import pytest import numpy as np import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.c_transforms as c_vision -import mindspore.dataset.transforms.vision.py_transforms as py_vision +import mindspore.dataset.transforms.py_transforms +import mindspore.dataset.vision.c_transforms as c_vision +import mindspore.dataset.vision.py_transforms as py_vision from mindspore import log as logger from util import diff_mse, visualize_image, save_and_check_md5, \ config_get_set_seed, config_get_set_num_parallel_workers @@ -42,9 +43,9 @@ def util_test_random_color_adjust_error(brightness=(1, 1), contrast=(1, 1), satu (lambda image: (image.transpose(1, 2, 0) * 255).astype(np.uint8)) ] - transform = py_vision.ComposeOp(transforms) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) - data1 = data1.map(input_columns=["image"], operations=transform()) + data1 = data1.map(input_columns=["image"], operations=transform) # if input is grayscale, the output dimensions should be single channel, the following should fail random_adjust_op = c_vision.RandomColorAdjust(brightness=brightness, contrast=contrast, saturation=saturation, @@ -86,9 +87,9 @@ def util_test_random_color_adjust_op(brightness=(1, 1), contrast=(1, 1), saturat hue=hue), py_vision.ToTensor() ] - transform = py_vision.ComposeOp(transforms) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) - data2 = data2.map(input_columns=["image"], operations=transform()) + data2 = data2.map(input_columns=["image"], operations=transform) num_iter = 0 for item1, item2 in zip(data1.create_dict_iterator(num_epochs=1), data2.create_dict_iterator(num_epochs=1)): @@ -211,9 +212,9 @@ def test_random_color_adjust_md5(): py_vision.RandomColorAdjust(0.4, 0.4, 0.4, 0.1), py_vision.ToTensor() ] - transform = py_vision.ComposeOp(transforms) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) - data2 = data2.map(input_columns=["image"], operations=transform()) + data2 = data2.map(input_columns=["image"], operations=transform) # Compare with expected md5 from images filename = "random_color_adjust_01_c_result.npz" save_and_check_md5(data1, filename, generate_golden=GENERATE_GOLDEN) diff --git a/tests/ut/python/dataset/test_random_crop.py b/tests/ut/python/dataset/test_random_crop.py index f83da40a3b..c1e0d67e64 100644 --- a/tests/ut/python/dataset/test_random_crop.py +++ b/tests/ut/python/dataset/test_random_crop.py @@ -16,9 +16,11 @@ Testing RandomCrop op in DE """ import numpy as np -import mindspore.dataset.transforms.vision.c_transforms as c_vision -import mindspore.dataset.transforms.vision.py_transforms as py_vision -import mindspore.dataset.transforms.vision.utils as mode + +import mindspore.dataset.transforms.py_transforms +import mindspore.dataset.vision.c_transforms as c_vision +import mindspore.dataset.vision.py_transforms as py_vision +import mindspore.dataset.vision.utils as mode import mindspore.dataset as ds from mindspore import log as logger from util import save_and_check_md5, visualize_list, config_get_set_seed, \ @@ -71,8 +73,8 @@ def test_random_crop_op_py(plot=False): py_vision.RandomCrop([512, 512], [200, 200, 200, 200]), py_vision.ToTensor() ] - transform1 = py_vision.ComposeOp(transforms1) - data1 = data1.map(input_columns=["image"], operations=transform1()) + transform1 = mindspore.dataset.transforms.py_transforms.Compose(transforms1) + data1 = data1.map(input_columns=["image"], operations=transform1) # Second dataset # Second dataset for comparison data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) @@ -80,8 +82,8 @@ def test_random_crop_op_py(plot=False): py_vision.Decode(), py_vision.ToTensor() ] - transform2 = py_vision.ComposeOp(transforms2) - data2 = data2.map(input_columns=["image"], operations=transform2()) + transform2 = mindspore.dataset.transforms.py_transforms.Compose(transforms2) + data2 = data2.map(input_columns=["image"], operations=transform2) crop_images = [] original_images = [] @@ -132,8 +134,8 @@ def test_random_crop_01_py(): py_vision.RandomCrop(512), py_vision.ToTensor() ] - transform = py_vision.ComposeOp(transforms) - data = data.map(input_columns=["image"], operations=transform()) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) + data = data.map(input_columns=["image"], operations=transform) filename = "random_crop_01_py_result.npz" save_and_check_md5(data, filename, generate_golden=GENERATE_GOLDEN) @@ -181,8 +183,8 @@ def test_random_crop_02_py(): py_vision.RandomCrop([512, 375]), py_vision.ToTensor() ] - transform = py_vision.ComposeOp(transforms) - data = data.map(input_columns=["image"], operations=transform()) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) + data = data.map(input_columns=["image"], operations=transform) filename = "random_crop_02_py_result.npz" save_and_check_md5(data, filename, generate_golden=GENERATE_GOLDEN) @@ -230,8 +232,8 @@ def test_random_crop_03_py(): py_vision.RandomCrop([2268, 4032]), py_vision.ToTensor() ] - transform = py_vision.ComposeOp(transforms) - data = data.map(input_columns=["image"], operations=transform()) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) + data = data.map(input_columns=["image"], operations=transform) filename = "random_crop_03_py_result.npz" save_and_check_md5(data, filename, generate_golden=GENERATE_GOLDEN) @@ -274,8 +276,8 @@ def test_random_crop_04_py(): py_vision.RandomCrop([2268, 4033]), py_vision.ToTensor() ] - transform = py_vision.ComposeOp(transforms) - data = data.map(input_columns=["image"], operations=transform()) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) + data = data.map(input_columns=["image"], operations=transform) try: data.create_dict_iterator(num_epochs=1).get_next() except RuntimeError as e: @@ -325,8 +327,8 @@ def test_random_crop_05_py(): py_vision.RandomCrop([2268, 4033], [200, 200, 200, 200], pad_if_needed=True), py_vision.ToTensor() ] - transform = py_vision.ComposeOp(transforms) - data = data.map(input_columns=["image"], operations=transform()) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) + data = data.map(input_columns=["image"], operations=transform) filename = "random_crop_05_py_result.npz" save_and_check_md5(data, filename, generate_golden=GENERATE_GOLDEN) @@ -370,8 +372,8 @@ def test_random_crop_06_py(): py_vision.RandomCrop([512, 512, 375]), py_vision.ToTensor() ] - transform = py_vision.ComposeOp(transforms) - data = data.map(input_columns=["image"], operations=transform()) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) + data = data.map(input_columns=["image"], operations=transform) except TypeError as e: logger.info("Got an exception in DE: {}".format(str(e))) assert "Size should be a single integer" in str(e) @@ -419,8 +421,8 @@ def test_random_crop_07_py(): py_vision.RandomCrop(512, [200, 200, 200, 200], fill_value=(255, 255, 255)), py_vision.ToTensor() ] - transform = py_vision.ComposeOp(transforms) - data = data.map(input_columns=["image"], operations=transform()) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) + data = data.map(input_columns=["image"], operations=transform) filename = "random_crop_07_py_result.npz" save_and_check_md5(data, filename, generate_golden=GENERATE_GOLDEN) @@ -470,8 +472,8 @@ def test_random_crop_08_py(): py_vision.RandomCrop(512, [200, 200, 200, 200], padding_mode=mode.Border.EDGE), py_vision.ToTensor() ] - transform = py_vision.ComposeOp(transforms) - data = data.map(input_columns=["image"], operations=transform()) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) + data = data.map(input_columns=["image"], operations=transform) filename = "random_crop_08_py_result.npz" save_and_check_md5(data, filename, generate_golden=GENERATE_GOLDEN) @@ -494,8 +496,8 @@ def test_random_crop_09(): # Note: if input is not PIL image, TypeError will raise py_vision.RandomCrop(512) ] - transform = py_vision.ComposeOp(transforms) - data = data.map(input_columns=["image"], operations=transform()) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) + data = data.map(input_columns=["image"], operations=transform) try: data.create_dict_iterator(num_epochs=1).get_next() except RuntimeError as e: @@ -523,8 +525,8 @@ def test_random_crop_comp(plot=False): py_vision.RandomCrop(cropped_size), py_vision.ToTensor() ] - transform = py_vision.ComposeOp(transforms) - data2 = data2.map(input_columns=["image"], operations=transform()) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) + data2 = data2.map(input_columns=["image"], operations=transform) image_c_cropped = [] image_py_cropped = [] diff --git a/tests/ut/python/dataset/test_random_crop_and_resize.py b/tests/ut/python/dataset/test_random_crop_and_resize.py index 5e5ed23da2..f701a3bd91 100644 --- a/tests/ut/python/dataset/test_random_crop_and_resize.py +++ b/tests/ut/python/dataset/test_random_crop_and_resize.py @@ -18,9 +18,10 @@ Testing RandomCropAndResize op in DE import numpy as np import cv2 -import mindspore.dataset.transforms.vision.c_transforms as c_vision -import mindspore.dataset.transforms.vision.py_transforms as py_vision -import mindspore.dataset.transforms.vision.utils as mode +import mindspore.dataset.transforms.py_transforms +import mindspore.dataset.vision.c_transforms as c_vision +import mindspore.dataset.vision.py_transforms as py_vision +import mindspore.dataset.vision.utils as mode import mindspore.dataset as ds from mindspore import log as logger from util import diff_mse, save_and_check_md5, visualize_list, \ @@ -80,8 +81,8 @@ def test_random_crop_and_resize_op_py(plot=False): py_vision.RandomResizedCrop((256, 512), (2, 2), (1, 3)), py_vision.ToTensor() ] - transform1 = py_vision.ComposeOp(transforms1) - data1 = data1.map(input_columns=["image"], operations=transform1()) + transform1 = mindspore.dataset.transforms.py_transforms.Compose(transforms1) + data1 = data1.map(input_columns=["image"], operations=transform1) # Second dataset # Second dataset for comparison data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) @@ -89,8 +90,8 @@ def test_random_crop_and_resize_op_py(plot=False): py_vision.Decode(), py_vision.ToTensor() ] - transform2 = py_vision.ComposeOp(transforms2) - data2 = data2.map(input_columns=["image"], operations=transform2()) + transform2 = mindspore.dataset.transforms.py_transforms.Compose(transforms2) + data2 = data2.map(input_columns=["image"], operations=transform2) num_iter = 0 crop_and_resize_images = [] original_images = [] @@ -131,8 +132,8 @@ def test_random_crop_and_resize_01(): py_vision.RandomResizedCrop((256, 512), (0.5, 0.5), (1, 1)), py_vision.ToTensor() ] - transform = py_vision.ComposeOp(transforms) - data2 = data2.map(input_columns=["image"], operations=transform()) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) + data2 = data2.map(input_columns=["image"], operations=transform) filename1 = "random_crop_and_resize_01_c_result.npz" filename2 = "random_crop_and_resize_01_py_result.npz" @@ -167,8 +168,8 @@ def test_random_crop_and_resize_02(): py_vision.RandomResizedCrop((256, 512), interpolation=mode.Inter.NEAREST), py_vision.ToTensor() ] - transform = py_vision.ComposeOp(transforms) - data2 = data2.map(input_columns=["image"], operations=transform()) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) + data2 = data2.map(input_columns=["image"], operations=transform) filename1 = "random_crop_and_resize_02_c_result.npz" filename2 = "random_crop_and_resize_02_py_result.npz" @@ -202,8 +203,8 @@ def test_random_crop_and_resize_03(): py_vision.RandomResizedCrop((256, 512), max_attempts=1), py_vision.ToTensor() ] - transform = py_vision.ComposeOp(transforms) - data2 = data2.map(input_columns=["image"], operations=transform()) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) + data2 = data2.map(input_columns=["image"], operations=transform) filename1 = "random_crop_and_resize_03_c_result.npz" filename2 = "random_crop_and_resize_03_py_result.npz" @@ -251,8 +252,8 @@ def test_random_crop_and_resize_04_py(): py_vision.RandomResizedCrop((256, 512), (1, 0.5), (0.5, 0.5)), py_vision.ToTensor() ] - transform = py_vision.ComposeOp(transforms) - data = data.map(input_columns=["image"], operations=transform()) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) + data = data.map(input_columns=["image"], operations=transform) except ValueError as e: logger.info("Got an exception in DE: {}".format(str(e))) assert "Input is not within the required interval of (0 to 16777216)." in str(e) @@ -294,8 +295,8 @@ def test_random_crop_and_resize_05_py(): py_vision.RandomResizedCrop((256, 512), (1, 1), (1, 0.5)), py_vision.ToTensor() ] - transform = py_vision.ComposeOp(transforms) - data = data.map(input_columns=["image"], operations=transform()) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) + data = data.map(input_columns=["image"], operations=transform) except ValueError as e: logger.info("Got an exception in DE: {}".format(str(e))) assert "Input is not within the required interval of (0 to 16777216)." in str(e) @@ -321,8 +322,8 @@ def test_random_crop_and_resize_comp(plot=False): py_vision.RandomResizedCrop(512, (1, 1), (0.5, 0.5)), py_vision.ToTensor() ] - transform = py_vision.ComposeOp(transforms) - data2 = data2.map(input_columns=["image"], operations=transform()) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) + data2 = data2.map(input_columns=["image"], operations=transform) image_c_cropped = [] image_py_cropped = [] diff --git a/tests/ut/python/dataset/test_random_crop_and_resize_with_bbox.py b/tests/ut/python/dataset/test_random_crop_and_resize_with_bbox.py index 5048c9dd08..6d3a24358a 100644 --- a/tests/ut/python/dataset/test_random_crop_and_resize_with_bbox.py +++ b/tests/ut/python/dataset/test_random_crop_and_resize_with_bbox.py @@ -17,7 +17,7 @@ Testing RandomCropAndResizeWithBBox op in DE """ import numpy as np import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.c_transforms as c_vision +import mindspore.dataset.vision.c_transforms as c_vision from mindspore import log as logger from util import visualize_with_bounding_boxes, InvalidBBoxType, check_bad_bbox, \ @@ -50,7 +50,7 @@ def test_random_resized_crop_with_bbox_op_c(plot_vis=False): # map to apply ops dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"], output_columns=["image", "bbox"], - columns_order=["image", "bbox"], + column_order=["image", "bbox"], operations=[test_op]) filename = "random_resized_crop_with_bbox_01_c_result.npz" @@ -87,7 +87,7 @@ def test_random_resized_crop_with_bbox_op_coco_c(plot_vis=False): dataCoco2 = dataCoco2.map(input_columns=["image", "bbox"], output_columns=["image", "bbox"], - columns_order=["image", "bbox"], + column_order=["image", "bbox"], operations=[test_op]) unaugSamp, augSamp = [], [] @@ -116,13 +116,13 @@ def test_random_resized_crop_with_bbox_op_edge_c(plot_vis=False): # maps to convert data into valid edge case data dataVoc1 = dataVoc1.map(input_columns=["image", "bbox"], output_columns=["image", "bbox"], - columns_order=["image", "bbox"], + column_order=["image", "bbox"], operations=[lambda img, bboxes: (img, np.array([[0, 0, img.shape[1], img.shape[0]]]).astype(bboxes.dtype))]) # Test Op added to list of Operations here dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"], output_columns=["image", "bbox"], - columns_order=["image", "bbox"], + column_order=["image", "bbox"], operations=[lambda img, bboxes: (img, np.array([[0, 0, img.shape[1], img.shape[0]]]).astype(bboxes.dtype)), test_op]) unaugSamp, augSamp = [], [] @@ -151,7 +151,7 @@ def test_random_resized_crop_with_bbox_op_invalid_c(): # map to apply ops dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"], output_columns=["image", "bbox"], - columns_order=["image", "bbox"], + column_order=["image", "bbox"], operations=[test_op]) for _ in dataVoc2.create_dict_iterator(num_epochs=1): @@ -177,7 +177,7 @@ def test_random_resized_crop_with_bbox_op_invalid2_c(): # map to apply ops dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"], output_columns=["image", "bbox"], - columns_order=["image", "bbox"], + column_order=["image", "bbox"], operations=[test_op]) for _ in dataVoc2.create_dict_iterator(num_epochs=1): diff --git a/tests/ut/python/dataset/test_random_crop_decode_resize.py b/tests/ut/python/dataset/test_random_crop_decode_resize.py index 35f9410c43..c2dc202c9c 100644 --- a/tests/ut/python/dataset/test_random_crop_decode_resize.py +++ b/tests/ut/python/dataset/test_random_crop_decode_resize.py @@ -16,7 +16,7 @@ Testing RandomCropDecodeResize op in DE """ import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.c_transforms as vision +import mindspore.dataset.vision.c_transforms as vision from mindspore import log as logger from util import diff_mse, visualize_image, save_and_check_md5, \ config_get_set_seed, config_get_set_num_parallel_workers diff --git a/tests/ut/python/dataset/test_random_crop_with_bbox.py b/tests/ut/python/dataset/test_random_crop_with_bbox.py index 186ce0fd63..ee38200f27 100644 --- a/tests/ut/python/dataset/test_random_crop_with_bbox.py +++ b/tests/ut/python/dataset/test_random_crop_with_bbox.py @@ -17,8 +17,8 @@ Testing RandomCropWithBBox op in DE """ import numpy as np import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.c_transforms as c_vision -import mindspore.dataset.transforms.vision.utils as mode +import mindspore.dataset.vision.c_transforms as c_vision +import mindspore.dataset.vision.utils as mode from mindspore import log as logger from util import visualize_with_bounding_boxes, InvalidBBoxType, check_bad_bbox, \ @@ -48,7 +48,7 @@ def test_random_crop_with_bbox_op_c(plot_vis=False): # map to apply ops dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"], output_columns=["image", "bbox"], - columns_order=["image", "bbox"], + column_order=["image", "bbox"], operations=[test_op]) # Add column for "bbox" unaugSamp, augSamp = [], [] @@ -78,7 +78,7 @@ def test_random_crop_with_bbox_op_coco_c(plot_vis=False): dataCoco2 = dataCoco2.map(input_columns=["image", "bbox"], output_columns=["image", "bbox"], - columns_order=["image", "bbox"], + column_order=["image", "bbox"], operations=[test_op]) unaugSamp, augSamp = [], [] @@ -110,7 +110,7 @@ def test_random_crop_with_bbox_op2_c(plot_vis=False): # map to apply ops dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"], output_columns=["image", "bbox"], - columns_order=["image", "bbox"], + column_order=["image", "bbox"], operations=[test_op]) filename = "random_crop_with_bbox_01_c_result.npz" @@ -147,7 +147,7 @@ def test_random_crop_with_bbox_op3_c(plot_vis=False): # map to apply ops dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"], output_columns=["image", "bbox"], - columns_order=["image", "bbox"], + column_order=["image", "bbox"], operations=[test_op]) unaugSamp, augSamp = [], [] @@ -177,14 +177,14 @@ def test_random_crop_with_bbox_op_edge_c(plot_vis=False): # maps to convert data into valid edge case data dataVoc1 = dataVoc1.map(input_columns=["image", "bbox"], output_columns=["image", "bbox"], - columns_order=["image", "bbox"], + column_order=["image", "bbox"], operations=[lambda img, bboxes: ( img, np.array([[0, 0, img.shape[1], img.shape[0]]]).astype(bboxes.dtype))]) # Test Op added to list of Operations here dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"], output_columns=["image", "bbox"], - columns_order=["image", "bbox"], + column_order=["image", "bbox"], operations=[lambda img, bboxes: ( img, np.array([[0, 0, img.shape[1], img.shape[0]]]).astype(bboxes.dtype)), test_op]) @@ -214,7 +214,7 @@ def test_random_crop_with_bbox_op_invalid_c(): # map to apply ops dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"], output_columns=["image", "bbox"], - columns_order=["image", "bbox"], + column_order=["image", "bbox"], operations=[test_op]) # Add column for "bbox" for _ in dataVoc2.create_dict_iterator(num_epochs=1): @@ -254,7 +254,7 @@ def test_random_crop_with_bbox_op_bad_padding(): dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"], output_columns=["image", "bbox"], - columns_order=["image", "bbox"], + column_order=["image", "bbox"], operations=[test_op]) for _ in dataVoc2.create_dict_iterator(num_epochs=1): @@ -268,7 +268,7 @@ def test_random_crop_with_bbox_op_bad_padding(): dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"], output_columns=["image", "bbox"], - columns_order=["image", "bbox"], + column_order=["image", "bbox"], operations=[test_op]) for _ in dataVoc2.create_dict_iterator(num_epochs=1): diff --git a/tests/ut/python/dataset/test_random_erasing.py b/tests/ut/python/dataset/test_random_erasing.py index b495239791..5dc3efd553 100644 --- a/tests/ut/python/dataset/test_random_erasing.py +++ b/tests/ut/python/dataset/test_random_erasing.py @@ -18,7 +18,8 @@ Testing RandomErasing op in DE import numpy as np import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.py_transforms as vision +import mindspore.dataset.transforms.py_transforms +import mindspore.dataset.vision.py_transforms as vision from mindspore import log as logger from util import diff_mse, visualize_image, save_and_check_md5, \ config_get_set_seed, config_get_set_num_parallel_workers @@ -41,8 +42,8 @@ def test_random_erasing_op(plot=False): vision.ToTensor(), vision.RandomErasing(value='random') ] - transform_1 = vision.ComposeOp(transforms_1) - data1 = data1.map(input_columns=["image"], operations=transform_1()) + transform_1 = mindspore.dataset.transforms.py_transforms.Compose(transforms_1) + data1 = data1.map(input_columns=["image"], operations=transform_1) # Second dataset data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) @@ -51,8 +52,8 @@ def test_random_erasing_op(plot=False): vision.ToTensor(), vision.Cutout(80) ] - transform_2 = vision.ComposeOp(transforms_2) - data2 = data2.map(input_columns=["image"], operations=transform_2()) + transform_2 = mindspore.dataset.transforms.py_transforms.Compose(transforms_2) + data2 = data2.map(input_columns=["image"], operations=transform_2) num_iter = 0 for item1, item2 in zip(data1.create_dict_iterator(num_epochs=1), data2.create_dict_iterator(num_epochs=1)): @@ -86,8 +87,8 @@ def test_random_erasing_md5(): vision.ToTensor(), vision.RandomErasing(value='random') ] - transform_1 = vision.ComposeOp(transforms_1) - data = data.map(input_columns=["image"], operations=transform_1()) + transform_1 = mindspore.dataset.transforms.py_transforms.Compose(transforms_1) + data = data.map(input_columns=["image"], operations=transform_1) # Compare with expected md5 from images filename = "random_erasing_01_result.npz" save_and_check_md5(data, filename, generate_golden=GENERATE_GOLDEN) diff --git a/tests/ut/python/dataset/test_random_grayscale.py b/tests/ut/python/dataset/test_random_grayscale.py index 75e1d18305..ed97521063 100644 --- a/tests/ut/python/dataset/test_random_grayscale.py +++ b/tests/ut/python/dataset/test_random_grayscale.py @@ -16,7 +16,9 @@ Testing RandomGrayscale op in DE """ import numpy as np -import mindspore.dataset.transforms.vision.py_transforms as py_vision + +import mindspore.dataset.transforms.py_transforms +import mindspore.dataset.vision.py_transforms as py_vision import mindspore.dataset as ds from mindspore import log as logger from util import save_and_check_md5, visualize_list, \ @@ -41,8 +43,8 @@ def test_random_grayscale_valid_prob(plot=False): py_vision.RandomGrayscale(1), py_vision.ToTensor() ] - transform1 = py_vision.ComposeOp(transforms1) - data1 = data1.map(input_columns=["image"], operations=transform1()) + transform1 = mindspore.dataset.transforms.py_transforms.Compose(transforms1) + data1 = data1.map(input_columns=["image"], operations=transform1) # Second dataset data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) @@ -50,8 +52,8 @@ def test_random_grayscale_valid_prob(plot=False): py_vision.Decode(), py_vision.ToTensor() ] - transform2 = py_vision.ComposeOp(transforms2) - data2 = data2.map(input_columns=["image"], operations=transform2()) + transform2 = mindspore.dataset.transforms.py_transforms.Compose(transforms2) + data2 = data2.map(input_columns=["image"], operations=transform2) image_gray = [] image = [] @@ -80,8 +82,8 @@ def test_random_grayscale_input_grayscale_images(): py_vision.RandomGrayscale(0.5), py_vision.ToTensor() ] - transform1 = py_vision.ComposeOp(transforms1) - data1 = data1.map(input_columns=["image"], operations=transform1()) + transform1 = mindspore.dataset.transforms.py_transforms.Compose(transforms1) + data1 = data1.map(input_columns=["image"], operations=transform1) # Second dataset data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) @@ -89,8 +91,8 @@ def test_random_grayscale_input_grayscale_images(): py_vision.Decode(), py_vision.ToTensor() ] - transform2 = py_vision.ComposeOp(transforms2) - data2 = data2.map(input_columns=["image"], operations=transform2()) + transform2 = mindspore.dataset.transforms.py_transforms.Compose(transforms2) + data2 = data2.map(input_columns=["image"], operations=transform2) image_gray = [] image = [] @@ -124,8 +126,8 @@ def test_random_grayscale_md5_valid_input(): py_vision.RandomGrayscale(0.8), py_vision.ToTensor() ] - transform = py_vision.ComposeOp(transforms) - data = data.map(input_columns=["image"], operations=transform()) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) + data = data.map(input_columns=["image"], operations=transform) # Check output images with md5 comparison filename = "random_grayscale_01_result.npz" @@ -150,8 +152,8 @@ def test_random_grayscale_md5_no_param(): py_vision.RandomGrayscale(), py_vision.ToTensor() ] - transform = py_vision.ComposeOp(transforms) - data = data.map(input_columns=["image"], operations=transform()) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) + data = data.map(input_columns=["image"], operations=transform) # Check output images with md5 comparison filename = "random_grayscale_02_result.npz" @@ -175,8 +177,8 @@ def test_random_grayscale_invalid_param(): py_vision.RandomGrayscale(1.5), py_vision.ToTensor() ] - transform = py_vision.ComposeOp(transforms) - data = data.map(input_columns=["image"], operations=transform()) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) + data = data.map(input_columns=["image"], operations=transform) except ValueError as e: logger.info("Got an exception in DE: {}".format(str(e))) assert "Input prob is not within the required interval of (0.0 to 1.0)." in str(e) diff --git a/tests/ut/python/dataset/test_random_horizontal_flip.py b/tests/ut/python/dataset/test_random_horizontal_flip.py index 9eac83cfb5..a88e6765be 100644 --- a/tests/ut/python/dataset/test_random_horizontal_flip.py +++ b/tests/ut/python/dataset/test_random_horizontal_flip.py @@ -17,8 +17,9 @@ Testing the random horizontal flip op in DE """ import numpy as np import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.c_transforms as c_vision -import mindspore.dataset.transforms.vision.py_transforms as py_vision +import mindspore.dataset.transforms.py_transforms +import mindspore.dataset.vision.c_transforms as c_vision +import mindspore.dataset.vision.py_transforms as py_vision from mindspore import log as logger from util import save_and_check_md5, visualize_list, visualize_image, diff_mse, \ config_get_set_seed, config_get_set_num_parallel_workers @@ -114,8 +115,8 @@ def test_random_horizontal_valid_prob_py(): py_vision.RandomHorizontalFlip(0.8), py_vision.ToTensor() ] - transform = py_vision.ComposeOp(transforms) - data = data.map(input_columns=["image"], operations=transform()) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) + data = data.map(input_columns=["image"], operations=transform) filename = "random_horizontal_01_py_result.npz" save_and_check_md5(data, filename, generate_golden=GENERATE_GOLDEN) @@ -160,8 +161,8 @@ def test_random_horizontal_invalid_prob_py(): py_vision.RandomHorizontalFlip(1.5), py_vision.ToTensor() ] - transform = py_vision.ComposeOp(transforms) - data = data.map(input_columns=["image"], operations=transform()) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) + data = data.map(input_columns=["image"], operations=transform) except ValueError as e: logger.info("Got an exception in DE: {}".format(str(e))) assert "Input prob is not within the required interval of (0.0 to 1.0)." in str(e) @@ -188,8 +189,8 @@ def test_random_horizontal_comp(plot=False): py_vision.RandomHorizontalFlip(1), py_vision.ToTensor() ] - transform = py_vision.ComposeOp(transforms) - data2 = data2.map(input_columns=["image"], operations=transform()) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) + data2 = data2.map(input_columns=["image"], operations=transform) images_list_c = [] images_list_py = [] diff --git a/tests/ut/python/dataset/test_random_horizontal_flip_with_bbox.py b/tests/ut/python/dataset/test_random_horizontal_flip_with_bbox.py index 5ec00fafb3..d12f715de3 100644 --- a/tests/ut/python/dataset/test_random_horizontal_flip_with_bbox.py +++ b/tests/ut/python/dataset/test_random_horizontal_flip_with_bbox.py @@ -18,7 +18,7 @@ Testing the random horizontal flip with bounding boxes op in DE import numpy as np import mindspore.log as logger import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.c_transforms as c_vision +import mindspore.dataset.vision.c_transforms as c_vision from util import visualize_with_bounding_boxes, InvalidBBoxType, check_bad_bbox, \ config_get_set_seed, config_get_set_num_parallel_workers, save_and_check_md5 @@ -47,7 +47,7 @@ def test_random_horizontal_flip_with_bbox_op_c(plot_vis=False): dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"], output_columns=["image", "bbox"], - columns_order=["image", "bbox"], + column_order=["image", "bbox"], operations=[test_op]) unaugSamp, augSamp = [], [] @@ -77,7 +77,7 @@ def test_random_horizontal_flip_with_bbox_op_coco_c(plot_vis=False): dataCoco2 = dataCoco2.map(input_columns=["image", "bbox"], output_columns=["image", "bbox"], - columns_order=["image", "bbox"], + column_order=["image", "bbox"], operations=[test_op]) unaugSamp, augSamp = [], [] @@ -113,7 +113,7 @@ def test_random_horizontal_flip_with_bbox_valid_rand_c(plot_vis=False): # map to apply ops dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"], output_columns=["image", "bbox"], - columns_order=["image", "bbox"], + column_order=["image", "bbox"], operations=[test_op]) filename = "random_horizontal_flip_with_bbox_01_c_result.npz" @@ -149,17 +149,17 @@ def test_random_horizontal_flip_with_bbox_valid_edge_c(plot_vis=False): # Add column for "bbox" dataVoc1 = dataVoc1.map(input_columns=["image", "bbox"], output_columns=["image", "bbox"], - columns_order=["image", "bbox"], + column_order=["image", "bbox"], operations=lambda img, bbox: (img, np.array([[0, 0, img.shape[1], img.shape[0], 0, 0, 0]]).astype(np.float32))) dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"], output_columns=["image", "bbox"], - columns_order=["image", "bbox"], + column_order=["image", "bbox"], operations=lambda img, bbox: (img, np.array([[0, 0, img.shape[1], img.shape[0], 0, 0, 0]]).astype(np.float32))) dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"], output_columns=["image", "bbox"], - columns_order=["image", "bbox"], + column_order=["image", "bbox"], operations=[test_op]) unaugSamp, augSamp = [], [] @@ -186,7 +186,7 @@ def test_random_horizontal_flip_with_bbox_invalid_prob_c(): # map to apply ops dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"], output_columns=["image", "bbox"], - columns_order=["image", "bbox"], + column_order=["image", "bbox"], operations=[test_op]) # Add column for "bbox" except ValueError as error: logger.info("Got an exception in DE: {}".format(str(error))) diff --git a/tests/ut/python/dataset/test_random_order.py b/tests/ut/python/dataset/test_random_order.py index d429f6b8f0..c882d822cc 100644 --- a/tests/ut/python/dataset/test_random_order.py +++ b/tests/ut/python/dataset/test_random_order.py @@ -17,7 +17,8 @@ Testing RandomOrder op in DE """ import numpy as np import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.py_transforms as py_vision +import mindspore.dataset.transforms.py_transforms +import mindspore.dataset.vision.py_transforms as py_vision from mindspore import log as logger from util import visualize_list, config_get_set_seed, \ config_get_set_num_parallel_workers, save_and_check_md5 @@ -40,20 +41,20 @@ def test_random_order_op(plot=False): py_vision.RandomOrder(transforms_list), py_vision.ToTensor() ] - transform1 = py_vision.ComposeOp(transforms1) + transform1 = mindspore.dataset.transforms.py_transforms.Compose(transforms1) transforms2 = [ py_vision.Decode(), py_vision.ToTensor() ] - transform2 = py_vision.ComposeOp(transforms2) + transform2 = mindspore.dataset.transforms.py_transforms.Compose(transforms2) # First dataset data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) - data1 = data1.map(input_columns=["image"], operations=transform1()) + data1 = data1.map(input_columns=["image"], operations=transform1) # Second dataset data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) - data2 = data2.map(input_columns=["image"], operations=transform2()) + data2 = data2.map(input_columns=["image"], operations=transform2) image_order = [] image_original = [] @@ -80,11 +81,11 @@ def test_random_order_md5(): py_vision.RandomOrder(transforms_list), py_vision.ToTensor() ] - transform = py_vision.ComposeOp(transforms) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) # Generate dataset data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) - data = data.map(input_columns=["image"], operations=transform()) + data = data.map(input_columns=["image"], operations=transform) # check results with md5 comparison filename = "random_order_01_result.npz" diff --git a/tests/ut/python/dataset/test_random_perspective.py b/tests/ut/python/dataset/test_random_perspective.py index 001a57dbd2..94d888a784 100644 --- a/tests/ut/python/dataset/test_random_perspective.py +++ b/tests/ut/python/dataset/test_random_perspective.py @@ -17,8 +17,9 @@ Testing RandomPerspective op in DE """ import numpy as np import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.py_transforms as py_vision -from mindspore.dataset.transforms.vision.utils import Inter +import mindspore.dataset.transforms.py_transforms +import mindspore.dataset.vision.py_transforms as py_vision +from mindspore.dataset.vision.utils import Inter from mindspore import log as logger from util import visualize_list, save_and_check_md5, \ config_get_set_seed, config_get_set_num_parallel_workers @@ -41,20 +42,20 @@ def test_random_perspective_op(plot=False): py_vision.RandomPerspective(), py_vision.ToTensor() ] - transform1 = py_vision.ComposeOp(transforms1) + transform1 = mindspore.dataset.transforms.py_transforms.Compose(transforms1) transforms2 = [ py_vision.Decode(), py_vision.ToTensor() ] - transform2 = py_vision.ComposeOp(transforms2) + transform2 = mindspore.dataset.transforms.py_transforms.Compose(transforms2) # First dataset data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) - data1 = data1.map(input_columns=["image"], operations=transform1()) + data1 = data1.map(input_columns=["image"], operations=transform1) # Second dataset data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) - data2 = data2.map(input_columns=["image"], operations=transform2()) + data2 = data2.map(input_columns=["image"], operations=transform2) image_perspective = [] image_original = [] @@ -83,11 +84,11 @@ def skip_test_random_perspective_md5(): py_vision.Resize(1450), # resize to a smaller size to prevent round-off error py_vision.ToTensor() ] - transform = py_vision.ComposeOp(transforms) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) # Generate dataset data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) - data = data.map(input_columns=["image"], operations=transform()) + data = data.map(input_columns=["image"], operations=transform) # check results with md5 comparison filename = "random_perspective_01_result.npz" diff --git a/tests/ut/python/dataset/test_random_posterize.py b/tests/ut/python/dataset/test_random_posterize.py index 27f22a4dcc..2d79b0bc1e 100644 --- a/tests/ut/python/dataset/test_random_posterize.py +++ b/tests/ut/python/dataset/test_random_posterize.py @@ -17,7 +17,7 @@ Testing RandomPosterize op in DE """ import numpy as np import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.c_transforms as c_vision +import mindspore.dataset.vision.c_transforms as c_vision from mindspore import log as logger from util import visualize_list, save_and_check_md5, \ config_get_set_seed, config_get_set_num_parallel_workers, diff_mse diff --git a/tests/ut/python/dataset/test_random_resize.py b/tests/ut/python/dataset/test_random_resize.py index e7cb6328bb..d3ffefd67c 100644 --- a/tests/ut/python/dataset/test_random_resize.py +++ b/tests/ut/python/dataset/test_random_resize.py @@ -16,7 +16,7 @@ Testing RandomResize op in DE """ import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.c_transforms as vision +import mindspore.dataset.vision.c_transforms as vision from mindspore import log as logger from util import visualize_list, save_and_check_md5, \ config_get_set_seed, config_get_set_num_parallel_workers diff --git a/tests/ut/python/dataset/test_random_resize_with_bbox.py b/tests/ut/python/dataset/test_random_resize_with_bbox.py index 64db0deab1..5612b47f31 100644 --- a/tests/ut/python/dataset/test_random_resize_with_bbox.py +++ b/tests/ut/python/dataset/test_random_resize_with_bbox.py @@ -17,7 +17,7 @@ Testing the random resize with bounding boxes op in DE """ import numpy as np import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.c_transforms as c_vision +import mindspore.dataset.vision.c_transforms as c_vision from mindspore import log as logger from util import visualize_with_bounding_boxes, InvalidBBoxType, check_bad_bbox, \ @@ -50,7 +50,7 @@ def test_random_resize_with_bbox_op_voc_c(plot_vis=False): # map to apply ops dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"], output_columns=["image", "bbox"], - columns_order=["image", "bbox"], + column_order=["image", "bbox"], operations=[test_op]) filename = "random_resize_with_bbox_op_01_c_voc_result.npz" @@ -93,7 +93,7 @@ def test_random_resize_with_bbox_op_rand_coco_c(plot_vis=False): dataCoco2 = dataCoco2.map(input_columns=["image", "bbox"], output_columns=["image", "bbox"], - columns_order=["image", "bbox"], + column_order=["image", "bbox"], operations=[test_op]) filename = "random_resize_with_bbox_op_01_c_coco_result.npz" @@ -131,13 +131,13 @@ def test_random_resize_with_bbox_op_edge_c(plot_vis=False): # maps to convert data into valid edge case data dataVoc1 = dataVoc1.map(input_columns=["image", "bbox"], output_columns=["image", "bbox"], - columns_order=["image", "bbox"], + column_order=["image", "bbox"], operations=[lambda img, bboxes: ( img, np.array([[0, 0, img.shape[1], img.shape[0]]]).astype(bboxes.dtype))]) dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"], output_columns=["image", "bbox"], - columns_order=["image", "bbox"], + column_order=["image", "bbox"], operations=[lambda img, bboxes: ( img, np.array([[0, 0, img.shape[1], img.shape[0]]]).astype(bboxes.dtype)), test_op]) diff --git a/tests/ut/python/dataset/test_random_rotation.py b/tests/ut/python/dataset/test_random_rotation.py index 84a5b1482a..4eede355ec 100644 --- a/tests/ut/python/dataset/test_random_rotation.py +++ b/tests/ut/python/dataset/test_random_rotation.py @@ -19,9 +19,10 @@ import numpy as np import cv2 import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.c_transforms as c_vision -import mindspore.dataset.transforms.vision.py_transforms as py_vision -from mindspore.dataset.transforms.vision.utils import Inter +import mindspore.dataset.transforms.py_transforms +import mindspore.dataset.vision.c_transforms as c_vision +import mindspore.dataset.vision.py_transforms as py_vision +from mindspore.dataset.vision.utils import Inter from mindspore import log as logger from util import visualize_image, visualize_list, diff_mse, save_and_check_md5, \ config_get_set_seed, config_get_set_num_parallel_workers @@ -31,6 +32,7 @@ SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json" GENERATE_GOLDEN = False + def test_random_rotation_op_c(plot=False): """ Test RandomRotation in c++ transformations op @@ -74,16 +76,16 @@ def test_random_rotation_op_py(plot=False): # First dataset data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) # use [90, 90] to force rotate 90 degrees, expand is set to be True to match output size - transform1 = py_vision.ComposeOp([py_vision.Decode(), - py_vision.RandomRotation((90, 90), expand=True), - py_vision.ToTensor()]) - data1 = data1.map(input_columns=["image"], operations=transform1()) + transform1 = mindspore.dataset.transforms.py_transforms.Compose([py_vision.Decode(), + py_vision.RandomRotation((90, 90), expand=True), + py_vision.ToTensor()]) + data1 = data1.map(input_columns=["image"], operations=transform1) # Second dataset data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) - transform2 = py_vision.ComposeOp([py_vision.Decode(), - py_vision.ToTensor()]) - data2 = data2.map(input_columns=["image"], operations=transform2()) + transform2 = mindspore.dataset.transforms.py_transforms.Compose([py_vision.Decode(), + py_vision.ToTensor()]) + data2 = data2.map(input_columns=["image"], operations=transform2) num_iter = 0 for item1, item2 in zip(data1.create_dict_iterator(num_epochs=1), data2.create_dict_iterator(num_epochs=1)): @@ -143,14 +145,14 @@ def test_random_rotation_md5(): # Second dataset data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False) - transform2 = py_vision.ComposeOp([py_vision.Decode(), - py_vision.RandomRotation((0, 90), - expand=True, - resample=Inter.BILINEAR, - center=(50, 50), - fill_value=150), - py_vision.ToTensor()]) - data2 = data2.map(input_columns=["image"], operations=transform2()) + transform2 = mindspore.dataset.transforms.py_transforms.Compose([py_vision.Decode(), + py_vision.RandomRotation((0, 90), + expand=True, + resample=Inter.BILINEAR, + center=(50, 50), + fill_value=150), + py_vision.ToTensor()]) + data2 = data2.map(input_columns=["image"], operations=transform2) # Compare with expected md5 from images filename1 = "random_rotation_01_c_result.npz" @@ -186,9 +188,9 @@ def test_rotation_diff(plot=False): py_vision.RandomRotation((45, 45)), py_vision.ToTensor(), ] - transform = py_vision.ComposeOp(transforms) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) - data2 = data2.map(input_columns=["image"], operations=transform()) + data2 = data2.map(input_columns=["image"], operations=transform) num_iter = 0 image_list_c, image_list_py = [], [] @@ -206,7 +208,7 @@ def test_rotation_diff(plot=False): logger.info("dtype of py_image: {}".format(py_image.dtype)) mse = diff_mse(c_image, py_image) - assert mse < 0.001 # Rounding error + assert mse < 0.001 # Rounding error if plot: visualize_list(image_list_c, image_list_py, visualize_mode=2) diff --git a/tests/ut/python/dataset/test_random_select_subpolicy.py b/tests/ut/python/dataset/test_random_select_subpolicy.py index 4b3ac83c47..8226ee6663 100644 --- a/tests/ut/python/dataset/test_random_select_subpolicy.py +++ b/tests/ut/python/dataset/test_random_select_subpolicy.py @@ -15,7 +15,7 @@ import mindspore.dataset as ds import mindspore.dataset.transforms.c_transforms as ops -import mindspore.dataset.transforms.vision.c_transforms as visions +import mindspore.dataset.vision.c_transforms as visions def test_random_select_subpolicy(): diff --git a/tests/ut/python/dataset/test_random_sharpness.py b/tests/ut/python/dataset/test_random_sharpness.py index 1d8eb6030e..838d23aaaf 100644 --- a/tests/ut/python/dataset/test_random_sharpness.py +++ b/tests/ut/python/dataset/test_random_sharpness.py @@ -18,8 +18,9 @@ Testing RandomSharpness op in DE import numpy as np import mindspore.dataset as ds import mindspore.dataset.engine as de -import mindspore.dataset.transforms.vision.py_transforms as F -import mindspore.dataset.transforms.vision.c_transforms as C +import mindspore.dataset.transforms.py_transforms +import mindspore.dataset.vision.py_transforms as F +import mindspore.dataset.vision.c_transforms as C from mindspore import log as logger from util import visualize_list, visualize_one_channel_dataset, diff_mse, save_and_check_md5, \ config_get_set_seed, config_get_set_num_parallel_workers @@ -37,14 +38,14 @@ def test_random_sharpness_py(degrees=(0.7, 0.7), plot=False): logger.info("Test RandomSharpness python op") # Original Images - data = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + data = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) - transforms_original = F.ComposeOp([F.Decode(), - F.Resize((224, 224)), - F.ToTensor()]) + transforms_original = mindspore.dataset.transforms.py_transforms.Compose([F.Decode(), + F.Resize((224, 224)), + F.ToTensor()]) ds_original = data.map(input_columns="image", - operations=transforms_original()) + operations=transforms_original) ds_original = ds_original.batch(512) @@ -57,19 +58,19 @@ def test_random_sharpness_py(degrees=(0.7, 0.7), plot=False): axis=0) # Random Sharpness Adjusted Images - data = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + data = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) py_op = F.RandomSharpness() if degrees is not None: py_op = F.RandomSharpness(degrees) - transforms_random_sharpness = F.ComposeOp([F.Decode(), - F.Resize((224, 224)), - py_op, - F.ToTensor()]) + transforms_random_sharpness = mindspore.dataset.transforms.py_transforms.Compose([F.Decode(), + F.Resize((224, 224)), + py_op, + F.ToTensor()]) ds_random_sharpness = data.map(input_columns="image", - operations=transforms_random_sharpness()) + operations=transforms_random_sharpness) ds_random_sharpness = ds_random_sharpness.batch(512) @@ -106,11 +107,11 @@ def test_random_sharpness_py_md5(): F.RandomSharpness((20.0, 25.0)), F.ToTensor() ] - transform = F.ComposeOp(transforms) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) # Generate dataset - data = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) - data = data.map(input_columns=["image"], operations=transform()) + data = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) + data = data.map(input_columns=["image"], operations=transform) # check results with md5 comparison filename = "random_sharpness_py_01_result.npz" @@ -129,7 +130,7 @@ def test_random_sharpness_c(degrees=(1.6, 1.6), plot=False): logger.info("Test RandomSharpness cpp op") # Original Images - data = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + data = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) transforms_original = [C.Decode(), C.Resize((224, 224))] @@ -148,7 +149,7 @@ def test_random_sharpness_c(degrees=(1.6, 1.6), plot=False): axis=0) # Random Sharpness Adjusted Images - data = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + data = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) c_op = C.RandomSharpness() if degrees is not None: @@ -197,7 +198,7 @@ def test_random_sharpness_c_md5(): ] # Generate dataset - data = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + data = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) data = data.map(input_columns=["image"], operations=transforms) # check results with md5 comparison @@ -216,7 +217,7 @@ def test_random_sharpness_c_py(degrees=(1.0, 1.0), plot=False): logger.info("Test RandomSharpness C and python Op") # RandomSharpness Images - data = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + data = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) data = data.map(input_columns=["image"], operations=[C.Decode(), C.Resize((200, 300))]) @@ -224,9 +225,9 @@ def test_random_sharpness_c_py(degrees=(1.0, 1.0), plot=False): python_op = F.RandomSharpness(degrees) c_op = C.RandomSharpness(degrees) - transforms_op = F.ComposeOp([lambda img: F.ToPIL()(img.astype(np.uint8)), - python_op, - np.array])() + transforms_op = mindspore.dataset.transforms.py_transforms.Compose([lambda img: F.ToPIL()(img.astype(np.uint8)), + python_op, + np.array]) ds_random_sharpness_py = data.map(input_columns="image", operations=transforms_op) @@ -242,7 +243,7 @@ def test_random_sharpness_c_py(degrees=(1.0, 1.0), plot=False): image, axis=0) - data = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + data = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) data = data.map(input_columns=["image"], operations=[C.Decode(), C.Resize((200, 300))]) @@ -305,7 +306,7 @@ def test_random_sharpness_invalid_params(): """ logger.info("Test RandomSharpness with invalid input parameters.") try: - data = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + data = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) data = data.map(input_columns=["image"], operations=[C.Decode(), C.Resize((224, 224)), @@ -315,7 +316,7 @@ def test_random_sharpness_invalid_params(): assert "tuple" in str(error) try: - data = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + data = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) data = data.map(input_columns=["image"], operations=[C.Decode(), C.Resize((224, 224)), @@ -325,7 +326,7 @@ def test_random_sharpness_invalid_params(): assert "interval" in str(error) try: - data = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + data = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) data = data.map(input_columns=["image"], operations=[C.Decode(), C.Resize((224, 224)), diff --git a/tests/ut/python/dataset/test_random_solarize_op.py b/tests/ut/python/dataset/test_random_solarize_op.py index f636f814b0..09a0a92c00 100644 --- a/tests/ut/python/dataset/test_random_solarize_op.py +++ b/tests/ut/python/dataset/test_random_solarize_op.py @@ -18,7 +18,7 @@ Testing RandomSolarizeOp op in DE import pytest import mindspore.dataset as ds import mindspore.dataset.engine as de -import mindspore.dataset.transforms.vision.c_transforms as vision +import mindspore.dataset.vision.c_transforms as vision from mindspore import log as logger from util import visualize_list, save_and_check_md5, config_get_set_seed, config_get_set_num_parallel_workers, \ visualize_one_channel_dataset diff --git a/tests/ut/python/dataset/test_random_vertical_flip.py b/tests/ut/python/dataset/test_random_vertical_flip.py index 8574bc6970..004b19fcd6 100644 --- a/tests/ut/python/dataset/test_random_vertical_flip.py +++ b/tests/ut/python/dataset/test_random_vertical_flip.py @@ -17,8 +17,9 @@ Testing the random vertical flip op in DE """ import numpy as np import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.c_transforms as c_vision -import mindspore.dataset.transforms.vision.py_transforms as py_vision +import mindspore.dataset.transforms.py_transforms +import mindspore.dataset.vision.c_transforms as c_vision +import mindspore.dataset.vision.py_transforms as py_vision from mindspore import log as logger from util import save_and_check_md5, visualize_list, visualize_image, diff_mse, \ config_get_set_seed, config_get_set_num_parallel_workers @@ -114,8 +115,8 @@ def test_random_vertical_valid_prob_py(): py_vision.RandomVerticalFlip(0.8), py_vision.ToTensor() ] - transform = py_vision.ComposeOp(transforms) - data = data.map(input_columns=["image"], operations=transform()) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) + data = data.map(input_columns=["image"], operations=transform) filename = "random_vertical_01_py_result.npz" save_and_check_md5(data, filename, generate_golden=GENERATE_GOLDEN) @@ -159,8 +160,8 @@ def test_random_vertical_invalid_prob_py(): py_vision.RandomVerticalFlip(1.5), py_vision.ToTensor() ] - transform = py_vision.ComposeOp(transforms) - data = data.map(input_columns=["image"], operations=transform()) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) + data = data.map(input_columns=["image"], operations=transform) except ValueError as e: logger.info("Got an exception in DE: {}".format(str(e))) assert 'Input prob is not within the required interval of (0.0 to 1.0).' in str(e) @@ -188,8 +189,8 @@ def test_random_vertical_comp(plot=False): py_vision.RandomVerticalFlip(1), py_vision.ToTensor() ] - transform = py_vision.ComposeOp(transforms) - data2 = data2.map(input_columns=["image"], operations=transform()) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) + data2 = data2.map(input_columns=["image"], operations=transform) images_list_c = [] images_list_py = [] diff --git a/tests/ut/python/dataset/test_random_vertical_flip_with_bbox.py b/tests/ut/python/dataset/test_random_vertical_flip_with_bbox.py index c978bf918c..248ba5c967 100644 --- a/tests/ut/python/dataset/test_random_vertical_flip_with_bbox.py +++ b/tests/ut/python/dataset/test_random_vertical_flip_with_bbox.py @@ -17,7 +17,7 @@ Testing RandomVerticalFlipWithBBox op in DE """ import numpy as np import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.c_transforms as c_vision +import mindspore.dataset.vision.c_transforms as c_vision from mindspore import log as logger from util import visualize_with_bounding_boxes, InvalidBBoxType, check_bad_bbox, \ @@ -48,7 +48,7 @@ def test_random_vertical_flip_with_bbox_op_c(plot_vis=False): # map to apply ops dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"], output_columns=["image", "bbox"], - columns_order=["image", "bbox"], + column_order=["image", "bbox"], operations=[test_op]) unaugSamp, augSamp = [], [] @@ -77,7 +77,7 @@ def test_random_vertical_flip_with_bbox_op_coco_c(plot_vis=False): dataCoco2 = dataCoco2.map(input_columns=["image", "bbox"], output_columns=["image", "bbox"], - columns_order=["image", "bbox"], + column_order=["image", "bbox"], operations=[test_op]) test_op = c_vision.RandomVerticalFlipWithBBox(1) @@ -113,7 +113,7 @@ def test_random_vertical_flip_with_bbox_op_rand_c(plot_vis=False): # map to apply ops dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"], output_columns=["image", "bbox"], - columns_order=["image", "bbox"], + column_order=["image", "bbox"], operations=[test_op]) filename = "random_vertical_flip_with_bbox_01_c_result.npz" @@ -150,13 +150,13 @@ def test_random_vertical_flip_with_bbox_op_edge_c(plot_vis=False): # maps to convert data into valid edge case data dataVoc1 = dataVoc1.map(input_columns=["image", "bbox"], output_columns=["image", "bbox"], - columns_order=["image", "bbox"], + column_order=["image", "bbox"], operations=[lambda img, bboxes: (img, np.array([[0, 0, img.shape[1], img.shape[0]]]).astype(bboxes.dtype))]) # Test Op added to list of Operations here dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"], output_columns=["image", "bbox"], - columns_order=["image", "bbox"], + column_order=["image", "bbox"], operations=[lambda img, bboxes: (img, np.array([[0, 0, img.shape[1], img.shape[0]]]).astype(bboxes.dtype)), test_op]) unaugSamp, augSamp = [], [] @@ -183,7 +183,7 @@ def test_random_vertical_flip_with_bbox_op_invalid_c(): # map to apply ops dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"], output_columns=["image", "bbox"], - columns_order=["image", "bbox"], + column_order=["image", "bbox"], operations=[test_op]) for _ in dataVoc2.create_dict_iterator(num_epochs=1): diff --git a/tests/ut/python/dataset/test_repeat.py b/tests/ut/python/dataset/test_repeat.py index 0d0567832a..3bf065361a 100644 --- a/tests/ut/python/dataset/test_repeat.py +++ b/tests/ut/python/dataset/test_repeat.py @@ -18,7 +18,7 @@ Test Repeat Op import numpy as np import pytest import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.c_transforms as vision +import mindspore.dataset.vision.c_transforms as vision from mindspore import log as logger from util import save_and_check_dict diff --git a/tests/ut/python/dataset/test_rescale_op.py b/tests/ut/python/dataset/test_rescale_op.py index a0fac86984..a20f296455 100644 --- a/tests/ut/python/dataset/test_rescale_op.py +++ b/tests/ut/python/dataset/test_rescale_op.py @@ -16,7 +16,7 @@ Testing the rescale op in DE """ import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.c_transforms as vision +import mindspore.dataset.vision.c_transforms as vision from mindspore import log as logger from util import visualize_image, diff_mse, save_and_check_md5 diff --git a/tests/ut/python/dataset/test_resize.py b/tests/ut/python/dataset/test_resize.py index 11c2efd8e5..1402c6f95e 100644 --- a/tests/ut/python/dataset/test_resize.py +++ b/tests/ut/python/dataset/test_resize.py @@ -17,8 +17,8 @@ Testing Resize op in DE """ import pytest import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.c_transforms as vision -from mindspore.dataset.transforms.vision.utils import Inter +import mindspore.dataset.vision.c_transforms as vision +from mindspore.dataset.vision.utils import Inter from mindspore import log as logger from util import visualize_list, save_and_check_md5, \ config_get_set_seed, config_get_set_num_parallel_workers diff --git a/tests/ut/python/dataset/test_resize_with_bbox.py b/tests/ut/python/dataset/test_resize_with_bbox.py index 95a0c56556..f5d9df049a 100644 --- a/tests/ut/python/dataset/test_resize_with_bbox.py +++ b/tests/ut/python/dataset/test_resize_with_bbox.py @@ -19,7 +19,7 @@ import numpy as np import pytest import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.c_transforms as c_vision +import mindspore.dataset.vision.c_transforms as c_vision from mindspore import log as logger from util import visualize_with_bounding_boxes, InvalidBBoxType, check_bad_bbox, \ save_and_check_md5 @@ -50,7 +50,7 @@ def test_resize_with_bbox_op_voc_c(plot_vis=False): # map to apply ops dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"], output_columns=["image", "bbox"], - columns_order=["image", "bbox"], + column_order=["image", "bbox"], operations=[test_op]) filename = "resize_with_bbox_op_01_c_voc_result.npz" @@ -87,7 +87,7 @@ def test_resize_with_bbox_op_coco_c(plot_vis=False): dataCOCO2 = dataCOCO2.map(input_columns=["image", "bbox"], output_columns=["image", "bbox"], - columns_order=["image", "bbox"], + column_order=["image", "bbox"], operations=[test_op]) filename = "resize_with_bbox_op_01_c_coco_result.npz" @@ -121,13 +121,13 @@ def test_resize_with_bbox_op_edge_c(plot_vis=False): # maps to convert data into valid edge case data dataVoc1 = dataVoc1.map(input_columns=["image", "bbox"], output_columns=["image", "bbox"], - columns_order=["image", "bbox"], + column_order=["image", "bbox"], operations=[lambda img, bboxes: ( img, np.array([[0, 0, img.shape[1], img.shape[0]]]).astype(bboxes.dtype))]) dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"], output_columns=["image", "bbox"], - columns_order=["image", "bbox"], + column_order=["image", "bbox"], operations=[lambda img, bboxes: ( img, np.array([[0, 0, img.shape[1], img.shape[0]]]).astype(bboxes.dtype)), test_op]) diff --git a/tests/ut/python/dataset/test_rgb_hsv.py b/tests/ut/python/dataset/test_rgb_hsv.py index ad36fe06d1..0bfc35be02 100644 --- a/tests/ut/python/dataset/test_rgb_hsv.py +++ b/tests/ut/python/dataset/test_rgb_hsv.py @@ -21,8 +21,9 @@ import numpy as np from numpy.testing import assert_allclose import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.py_transforms as vision -import mindspore.dataset.transforms.vision.py_transforms_util as util +import mindspore.dataset.transforms.py_transforms +import mindspore.dataset.vision.py_transforms as vision +import mindspore.dataset.vision.py_transforms_util as util DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json" @@ -137,9 +138,9 @@ def test_rgb_hsv_pipeline(): vision.Resize([64, 64]), vision.ToTensor() ] - transforms1 = vision.ComposeOp(transforms1) + transforms1 = mindspore.dataset.transforms.py_transforms.Compose(transforms1) ds1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) - ds1 = ds1.map(input_columns=["image"], operations=transforms1()) + ds1 = ds1.map(input_columns=["image"], operations=transforms1) # Second dataset transforms2 = [ @@ -149,9 +150,9 @@ def test_rgb_hsv_pipeline(): vision.RgbToHsv(), vision.HsvToRgb() ] - transform2 = vision.ComposeOp(transforms2) + transform2 = mindspore.dataset.transforms.py_transforms.Compose(transforms2) ds2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) - ds2 = ds2.map(input_columns=["image"], operations=transform2()) + ds2 = ds2.map(input_columns=["image"], operations=transform2) num_iter = 0 for data1, data2 in zip(ds1.create_dict_iterator(num_epochs=1), ds2.create_dict_iterator(num_epochs=1)): diff --git a/tests/ut/python/dataset/test_serdes_dataset.py b/tests/ut/python/dataset/test_serdes_dataset.py index 422070a12d..c1d9824f49 100644 --- a/tests/ut/python/dataset/test_serdes_dataset.py +++ b/tests/ut/python/dataset/test_serdes_dataset.py @@ -26,9 +26,9 @@ from util import config_get_set_num_parallel_workers import mindspore.dataset as ds import mindspore.dataset.transforms.c_transforms as c -import mindspore.dataset.transforms.vision.c_transforms as vision +import mindspore.dataset.vision.c_transforms as vision from mindspore import log as logger -from mindspore.dataset.transforms.vision import Inter +from mindspore.dataset.vision import Inter @@ -47,7 +47,7 @@ def test_imagefolder(remove_json_files=True): # Constructing DE pipeline sampler = ds.WeightedRandomSampler(weights, 11) - data1 = ds.ImageFolderDatasetV2(data_dir, sampler=sampler) + data1 = ds.ImageFolderDataset(data_dir, sampler=sampler) data1 = data1.repeat(1) data1 = data1.map(input_columns=["image"], operations=[vision.Decode(True)]) rescale_op = vision.Rescale(rescale, shift) diff --git a/tests/ut/python/dataset/test_skip.py b/tests/ut/python/dataset/test_skip.py index 16d7c369ba..98fefed67e 100644 --- a/tests/ut/python/dataset/test_skip.py +++ b/tests/ut/python/dataset/test_skip.py @@ -16,7 +16,7 @@ import numpy as np import pytest import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.c_transforms as vision +import mindspore.dataset.vision.c_transforms as vision from mindspore import log as logger diff --git a/tests/ut/python/dataset/test_soft_dvpp.py b/tests/ut/python/dataset/test_soft_dvpp.py index 82556f8d05..d26ea716f7 100644 --- a/tests/ut/python/dataset/test_soft_dvpp.py +++ b/tests/ut/python/dataset/test_soft_dvpp.py @@ -16,7 +16,7 @@ Testing soft dvpp SoftDvppDecodeResizeJpeg and SoftDvppDecodeRandomCropResizeJpeg in DE """ import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.c_transforms as vision +import mindspore.dataset.vision.c_transforms as vision from mindspore import log as logger from util import diff_mse, visualize_image diff --git a/tests/ut/python/dataset/test_ten_crop.py b/tests/ut/python/dataset/test_ten_crop.py index 86d23cbf81..6728868d35 100644 --- a/tests/ut/python/dataset/test_ten_crop.py +++ b/tests/ut/python/dataset/test_ten_crop.py @@ -18,7 +18,8 @@ import pytest import numpy as np import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.py_transforms as vision +import mindspore.dataset.transforms.py_transforms +import mindspore.dataset.vision.py_transforms as vision from mindspore import log as logger from util import visualize_list, save_and_check_md5 @@ -37,8 +38,8 @@ def util_test_ten_crop(crop_size, vertical_flip=False, plot=False): vision.Decode(), vision.ToTensor(), ] - transform_1 = vision.ComposeOp(transforms_1) - data1 = data1.map(input_columns=["image"], operations=transform_1()) + transform_1 = mindspore.dataset.transforms.py_transforms.Compose(transforms_1) + data1 = data1.map(input_columns=["image"], operations=transform_1) # Second dataset data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) @@ -47,8 +48,8 @@ def util_test_ten_crop(crop_size, vertical_flip=False, plot=False): vision.TenCrop(crop_size, use_vertical_flip=vertical_flip), lambda images: np.stack([vision.ToTensor()(image) for image in images]) # 4D stack of 10 images ] - transform_2 = vision.ComposeOp(transforms_2) - data2 = data2.map(input_columns=["image"], operations=transform_2()) + transform_2 = mindspore.dataset.transforms.py_transforms.Compose(transforms_2) + data2 = data2.map(input_columns=["image"], operations=transform_2) num_iter = 0 for item1, item2 in zip(data1.create_dict_iterator(num_epochs=1), data2.create_dict_iterator(num_epochs=1)): num_iter += 1 @@ -109,8 +110,8 @@ def test_ten_crop_md5(): vision.TenCrop((200, 100), use_vertical_flip=True), lambda images: np.stack([vision.ToTensor()(image) for image in images]) # 4D stack of 10 images ] - transform_2 = vision.ComposeOp(transforms_2) - data2 = data2.map(input_columns=["image"], operations=transform_2()) + transform_2 = mindspore.dataset.transforms.py_transforms.Compose(transforms_2) + data2 = data2.map(input_columns=["image"], operations=transform_2) # Compare with expected md5 from images filename = "ten_crop_01_result.npz" save_and_check_md5(data2, filename, generate_golden=GENERATE_GOLDEN) @@ -169,8 +170,8 @@ def test_ten_crop_wrong_img_error_msg(): vision.TenCrop(200), vision.ToTensor() ] - transform = vision.ComposeOp(transforms) - data = data.map(input_columns=["image"], operations=transform()) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) + data = data.map(input_columns=["image"], operations=transform) with pytest.raises(RuntimeError) as info: data.create_tuple_iterator(num_epochs=1).get_next() diff --git a/tests/ut/python/dataset/test_text_basic_tokenizer.py b/tests/ut/python/dataset/test_text_basic_tokenizer.py index 3e143ad698..d5879b70f3 100644 --- a/tests/ut/python/dataset/test_text_basic_tokenizer.py +++ b/tests/ut/python/dataset/test_text_basic_tokenizer.py @@ -106,7 +106,7 @@ def check_basic_tokenizer_with_offsets(first, last, expected_tokens, expected_of with_offsets=True) dataset = dataset.map(input_columns=['text'], output_columns=['token', 'offsets_start', 'offsets_limit'], - columns_order=['token', 'offsets_start', 'offsets_limit'], operations=basic_tokenizer) + column_order=['token', 'offsets_start', 'offsets_limit'], operations=basic_tokenizer) count = 0 for i in dataset.create_dict_iterator(num_epochs=1): token = text.to_str(i['token']) diff --git a/tests/ut/python/dataset/test_text_bert_tokenizer.py b/tests/ut/python/dataset/test_text_bert_tokenizer.py index a858cd2dc1..33fecaca0f 100644 --- a/tests/ut/python/dataset/test_text_bert_tokenizer.py +++ b/tests/ut/python/dataset/test_text_bert_tokenizer.py @@ -213,7 +213,7 @@ def check_bert_tokenizer_with_offsets(first, last, expect_str, unknown_token=unknown_token, lower_case=lower_case, keep_whitespace=keep_whitespace, normalization_form=normalization_form, preserve_unused_token=preserve_unused_token, with_offsets=True) dataset = dataset.map(input_columns=['text'], output_columns=['token', 'offsets_start', 'offsets_limit'], - columns_order=['token', 'offsets_start', 'offsets_limit'], operations=tokenizer_op) + column_order=['token', 'offsets_start', 'offsets_limit'], operations=tokenizer_op) count = 0 for i in dataset.create_dict_iterator(num_epochs=1): token = text.to_str(i['token']) diff --git a/tests/ut/python/dataset/test_text_jieba_tokenizer.py b/tests/ut/python/dataset/test_text_jieba_tokenizer.py index 7a1518ff9e..c334bbbbed 100644 --- a/tests/ut/python/dataset/test_text_jieba_tokenizer.py +++ b/tests/ut/python/dataset/test_text_jieba_tokenizer.py @@ -202,7 +202,7 @@ def test_jieba_with_offsets_1(): data = ds.TextFileDataset(DATA_FILE) jieba_op = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.MP, with_offsets=True) data = data.map(input_columns=["text"], output_columns=["token", "offsets_start", "offsets_limit"], - columns_order=["token", "offsets_start", "offsets_limit"], + column_order=["token", "offsets_start", "offsets_limit"], operations=jieba_op, num_parallel_workers=1) expect = ['今天天气', '太好了', '我们', '一起', '去', '外面', '玩吧'] expected_offsets_start = [0, 12, 21, 27, 33, 36, 42] @@ -223,7 +223,7 @@ def test_jieba_with_offsets_1_1(): data = ds.TextFileDataset(DATA_FILE) jieba_op = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.HMM, with_offsets=True) data = data.map(input_columns=["text"], output_columns=["token", "offsets_start", "offsets_limit"], - columns_order=["token", "offsets_start", "offsets_limit"], + column_order=["token", "offsets_start", "offsets_limit"], operations=jieba_op, num_parallel_workers=1) expect = ['今天', '天气', '太', '好', '了', '我们', '一起', '去', '外面', '玩', '吧'] expected_offsets_start = [0, 6, 12, 15, 18, 21, 27, 33, 36, 42, 45] @@ -243,7 +243,7 @@ def test_jieba_with_offsets_1_2(): data = ds.TextFileDataset(DATA_FILE) jieba_op = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.MIX, with_offsets=True) data = data.map(input_columns=["text"], output_columns=["token", "offsets_start", "offsets_limit"], - columns_order=["token", "offsets_start", "offsets_limit"], + column_order=["token", "offsets_start", "offsets_limit"], operations=jieba_op, num_parallel_workers=1) expect = ['今天天气', '太好了', '我们', '一起', '去', '外面', '玩吧'] expected_offsets_start = [0, 12, 21, 27, 33, 36, 42] @@ -266,7 +266,7 @@ def test_jieba_with_offsets_2(): jieba_op.add_word("男默女泪") expect = ['男默女泪', '市', '长江大桥'] data = data.map(input_columns=["text"], output_columns=["token", "offsets_start", "offsets_limit"], - columns_order=["token", "offsets_start", "offsets_limit"], + column_order=["token", "offsets_start", "offsets_limit"], operations=jieba_op, num_parallel_workers=2) expected_offsets_start = [0, 12, 15] expected_offsets_limit = [12, 15, 27] @@ -287,7 +287,7 @@ def test_jieba_with_offsets_2_1(): jieba_op = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.MP, with_offsets=True) jieba_op.add_word("男默女泪", 10) data = data.map(input_columns=["text"], output_columns=["token", "offsets_start", "offsets_limit"], - columns_order=["token", "offsets_start", "offsets_limit"], + column_order=["token", "offsets_start", "offsets_limit"], operations=jieba_op, num_parallel_workers=2) expect = ['男默女泪', '市', '长江大桥'] expected_offsets_start = [0, 12, 15] @@ -309,7 +309,7 @@ def test_jieba_with_offsets_2_2(): jieba_op = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.MP, with_offsets=True) jieba_op.add_word("江大桥", 20000) data = data.map(input_columns=["text"], output_columns=["token", "offsets_start", "offsets_limit"], - columns_order=["token", "offsets_start", "offsets_limit"], + column_order=["token", "offsets_start", "offsets_limit"], operations=jieba_op, num_parallel_workers=2) expect = ['江州', '市长', '江大桥', '参加', '了', '长江大桥', '的', '通车', '仪式'] expected_offsets_start = [0, 6, 12, 21, 27, 30, 42, 45, 51] @@ -334,7 +334,7 @@ def test_jieba_with_offsets_3(): jieba_op = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.MP, with_offsets=True) jieba_op.add_dict(user_dict) data = data.map(input_columns=["text"], output_columns=["token", "offsets_start", "offsets_limit"], - columns_order=["token", "offsets_start", "offsets_limit"], + column_order=["token", "offsets_start", "offsets_limit"], operations=jieba_op, num_parallel_workers=1) expect = ['男默女泪', '市', '长江大桥'] expected_offsets_start = [0, 12, 15] @@ -360,7 +360,7 @@ def test_jieba_with_offsets_3_1(): jieba_op = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.MP, with_offsets=True) jieba_op.add_dict(user_dict) data = data.map(input_columns=["text"], output_columns=["token", "offsets_start", "offsets_limit"], - columns_order=["token", "offsets_start", "offsets_limit"], + column_order=["token", "offsets_start", "offsets_limit"], operations=jieba_op, num_parallel_workers=1) expect = ['男默女泪', '市长', '江大桥'] expected_offsets_start = [0, 12, 18] @@ -383,7 +383,7 @@ def test_jieba_with_offsets_4(): jieba_op = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.MP, with_offsets=True) jieba_op.add_dict(DICT_FILE) data = data.map(input_columns=["text"], output_columns=["token", "offsets_start", "offsets_limit"], - columns_order=["token", "offsets_start", "offsets_limit"], + column_order=["token", "offsets_start", "offsets_limit"], operations=jieba_op, num_parallel_workers=1) expect = ['今天天气', '太好了', '我们', '一起', '去', '外面', '玩吧'] expected_offsets_start = [0, 12, 21, 27, 33, 36, 42] @@ -406,7 +406,7 @@ def test_jieba_with_offsets_5(): jieba_op = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.MP, with_offsets=True) jieba_op.add_word("江大桥", 20000) data = data.map(input_columns=["text"], output_columns=["token", "offsets_start", "offsets_limit"], - columns_order=["token", "offsets_start", "offsets_limit"], + column_order=["token", "offsets_start", "offsets_limit"], operations=jieba_op, num_parallel_workers=1) expect = ['江州', '市长', '江大桥', '参加', '了', '长江大桥', '的', '通车', '仪式'] expected_offsets_start = [0, 6, 12, 21, 27, 30, 42, 45, 51] diff --git a/tests/ut/python/dataset/test_text_tokenizer.py b/tests/ut/python/dataset/test_text_tokenizer.py index f9854ef111..972a551423 100644 --- a/tests/ut/python/dataset/test_text_tokenizer.py +++ b/tests/ut/python/dataset/test_text_tokenizer.py @@ -60,7 +60,7 @@ def test_unicode_char_tokenizer_with_offsets(): dataset = ds.TextFileDataset(DATA_FILE, shuffle=False) tokenizer = text.UnicodeCharTokenizer(with_offsets=True) dataset = dataset.map(input_columns=['text'], output_columns=['token', 'offsets_start', 'offsets_limit'], - columns_order=['token', 'offsets_start', 'offsets_limit'], operations=tokenizer) + column_order=['token', 'offsets_start', 'offsets_limit'], operations=tokenizer) tokens = [] expected_offsets_start = [[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18], [0, 3, 6, 9, 12, 15], [0, 3, 6, 9, 10, 11, 12, 13, 14, 15, 16], [0, 1]] @@ -107,7 +107,7 @@ def test_whitespace_tokenizer_with_offsets(): dataset = ds.TextFileDataset(DATA_FILE, shuffle=False) tokenizer = text.WhitespaceTokenizer(with_offsets=True) dataset = dataset.map(input_columns=['text'], output_columns=['token', 'offsets_start', 'offsets_limit'], - columns_order=['token', 'offsets_start', 'offsets_limit'], operations=tokenizer) + column_order=['token', 'offsets_start', 'offsets_limit'], operations=tokenizer) tokens = [] expected_offsets_start = [[0, 8, 11], [0], [0], [0]] expected_offsets_limit = [[7, 10, 19], [18], [17], [0]] @@ -173,7 +173,7 @@ def test_unicode_script_tokenizer_with_offsets(): dataset = ds.TextFileDataset(DATA_FILE, shuffle=False) tokenizer = text.UnicodeScriptTokenizer(keep_whitespace=False, with_offsets=True) dataset = dataset.map(input_columns=['text'], output_columns=['token', 'offsets_start', 'offsets_limit'], - columns_order=['token', 'offsets_start', 'offsets_limit'], operations=tokenizer) + column_order=['token', 'offsets_start', 'offsets_limit'], operations=tokenizer) tokens = [] expected_offsets_start = [[0, 8, 11, 18], [0, 15], [0, 9, 16], [0]] expected_offsets_limit = [[7, 10, 18, 19], [15, 18], [9, 16, 17], [0]] @@ -199,7 +199,7 @@ def test_unicode_script_tokenizer_with_offsets2(): dataset = ds.TextFileDataset(DATA_FILE, shuffle=False) tokenizer = text.UnicodeScriptTokenizer(keep_whitespace=True, with_offsets=True) dataset = dataset.map(input_columns=['text'], output_columns=['token', 'offsets_start', 'offsets_limit'], - columns_order=['token', 'offsets_start', 'offsets_limit'], operations=tokenizer) + column_order=['token', 'offsets_start', 'offsets_limit'], operations=tokenizer) tokens = [] expected_offsets_start = [[0, 7, 8, 10, 11, 18], [0, 15], [0, 9, 16], [0]] expected_offsets_limit = [[7, 8, 10, 11, 18, 19], [15, 18], [9, 16, 17], [2]] @@ -340,7 +340,7 @@ def test_regex_tokenizer_with_offsets(): dataset = dataset.take(last - first + 1) tokenizer_op = text.RegexTokenizer(delim_pattern, keep_delim_pattern, with_offsets=True) dataset = dataset.map(input_columns=['text'], output_columns=['token', 'offsets_start', 'offsets_limit'], - columns_order=['token', 'offsets_start', 'offsets_limit'], operations=tokenizer_op) + column_order=['token', 'offsets_start', 'offsets_limit'], operations=tokenizer_op) out_text = [] count = 0 for i in dataset.create_dict_iterator(num_epochs=1): diff --git a/tests/ut/python/dataset/test_text_wordpiece_tokenizer.py b/tests/ut/python/dataset/test_text_wordpiece_tokenizer.py index 988ba4b7a6..64b290c829 100644 --- a/tests/ut/python/dataset/test_text_wordpiece_tokenizer.py +++ b/tests/ut/python/dataset/test_text_wordpiece_tokenizer.py @@ -127,7 +127,7 @@ def check_wordpiece_tokenizer_with_offsets(first, last, expect_str, expected_off tokenizer_op = text.WordpieceTokenizer(vocab=vocab, with_offsets=True, unknown_token=unknown_token, max_bytes_per_token=max_bytes_per_token) dataset = dataset.map(input_columns=['text'], output_columns=['token', 'offsets_start', 'offsets_limit'], - columns_order=['token', 'offsets_start', 'offsets_limit'], operations=tokenizer_op) + column_order=['token', 'offsets_start', 'offsets_limit'], operations=tokenizer_op) count = 0 for i in dataset.create_dict_iterator(num_epochs=1): token = text.to_str(i['token']) diff --git a/tests/ut/python/dataset/test_to_pil.py b/tests/ut/python/dataset/test_to_pil.py index 2e260e5517..6060fda8ef 100644 --- a/tests/ut/python/dataset/test_to_pil.py +++ b/tests/ut/python/dataset/test_to_pil.py @@ -16,8 +16,9 @@ Testing ToPIL op in DE """ import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.c_transforms as c_vision -import mindspore.dataset.transforms.vision.py_transforms as py_vision +import mindspore.dataset.transforms.py_transforms +import mindspore.dataset.vision.c_transforms as c_vision +import mindspore.dataset.vision.py_transforms as py_vision from mindspore import log as logger from util import save_and_check_md5 @@ -43,8 +44,8 @@ def test_to_pil_01(): py_vision.CenterCrop(375), py_vision.ToTensor() ] - transform = py_vision.ComposeOp(transforms) - data1 = data1.map(input_columns=["image"], operations=transform()) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) + data1 = data1.map(input_columns=["image"], operations=transform) # Compare with expected md5 from images filename = "to_pil_01_result.npz" @@ -66,9 +67,9 @@ def test_to_pil_02(): py_vision.CenterCrop(375), py_vision.ToTensor() ] - transform = py_vision.ComposeOp(transforms) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) data1 = data1.map(input_columns=["image"], operations=decode_op) - data1 = data1.map(input_columns=["image"], operations=transform()) + data1 = data1.map(input_columns=["image"], operations=transform) # Compare with expected md5 from images filename = "to_pil_02_result.npz" diff --git a/tests/ut/python/dataset/test_to_type.py b/tests/ut/python/dataset/test_to_type.py index a1a01a9856..9ed668271a 100644 --- a/tests/ut/python/dataset/test_to_type.py +++ b/tests/ut/python/dataset/test_to_type.py @@ -17,7 +17,8 @@ Testing ToType op in DE """ import numpy as np import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.py_transforms as py_vision +import mindspore.dataset.transforms.py_transforms +import mindspore.dataset.vision.py_transforms as py_vision from mindspore import log as logger from util import save_and_check_md5 @@ -41,8 +42,8 @@ def test_to_type_op(): # Note: Convert the datatype from float32 to int16 py_vision.ToType(np.int16) ] - transform1 = py_vision.ComposeOp(transforms1) - data1 = data1.map(input_columns=["image"], operations=transform1()) + transform1 = mindspore.dataset.transforms.py_transforms.Compose(transforms1) + data1 = data1.map(input_columns=["image"], operations=transform1) # Second dataset data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) @@ -50,8 +51,8 @@ def test_to_type_op(): py_vision.Decode(), py_vision.ToTensor() ] - transform2 = py_vision.ComposeOp(transforms2) - data2 = data2.map(input_columns=["image"], operations=transform2()) + transform2 = mindspore.dataset.transforms.py_transforms.Compose(transforms2) + data2 = data2.map(input_columns=["image"], operations=transform2) for item1, item2 in zip(data1.create_dict_iterator(num_epochs=1), data2.create_dict_iterator(num_epochs=1)): image1 = item1["image"] @@ -78,8 +79,8 @@ def test_to_type_01(): # Note: Convert the datatype from float32 to int32 py_vision.ToType(np.int32) ] - transform = py_vision.ComposeOp(transforms) - data = data.map(input_columns=["image"], operations=transform()) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) + data = data.map(input_columns=["image"], operations=transform) # Compare with expected md5 from images filename = "to_type_01_result.npz" @@ -99,8 +100,8 @@ def test_to_type_02(): # Note: Convert to type int py_vision.ToType('int') ] - transform = py_vision.ComposeOp(transforms) - data = data.map(input_columns=["image"], operations=transform()) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) + data = data.map(input_columns=["image"], operations=transform) # Compare with expected md5 from images filename = "to_type_02_result.npz" @@ -121,8 +122,8 @@ def test_to_type_03(): # Note: If the object is not numpy, e.g. PIL image, TypeError will raise py_vision.ToType(np.int32) ] - transform = py_vision.ComposeOp(transforms) - data = data.map(input_columns=["image"], operations=transform()) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) + data = data.map(input_columns=["image"], operations=transform) except Exception as e: logger.info("Got an exception in DE: {}".format(str(e))) assert "Numpy" in str(e) @@ -143,8 +144,8 @@ def test_to_type_04(): # Note: if output_type is not explicitly given py_vision.ToType() ] - transform = py_vision.ComposeOp(transforms) - data = data.map(input_columns=["image"], operations=transform()) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) + data = data.map(input_columns=["image"], operations=transform) except Exception as e: logger.info("Got an exception in DE: {}".format(str(e))) assert "missing" in str(e) @@ -165,8 +166,8 @@ def test_to_type_05(): # Note: if output_type is not explicitly given py_vision.ToType('invalid') ] - transform = py_vision.ComposeOp(transforms) - data = data.map(input_columns=["image"], operations=transform()) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) + data = data.map(input_columns=["image"], operations=transform) except Exception as e: logger.info("Got an exception in DE: {}".format(str(e))) assert "data type" in str(e) diff --git a/tests/ut/python/dataset/test_type_cast.py b/tests/ut/python/dataset/test_type_cast.py index b47c874d1b..0547e4810d 100644 --- a/tests/ut/python/dataset/test_type_cast.py +++ b/tests/ut/python/dataset/test_type_cast.py @@ -20,8 +20,9 @@ import numpy as np import mindspore.common.dtype as mstype import mindspore.dataset as ds import mindspore.dataset.transforms.c_transforms as data_util -import mindspore.dataset.transforms.vision.c_transforms as c_vision -import mindspore.dataset.transforms.vision.py_transforms as py_vision +import mindspore.dataset.transforms.py_transforms +import mindspore.dataset.vision.c_transforms as c_vision +import mindspore.dataset.vision.py_transforms as py_vision from mindspore import log as logger DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] @@ -50,9 +51,9 @@ def test_type_cast(): transforms = [py_vision.Decode(), py_vision.ToTensor() ] - transform = py_vision.ComposeOp(transforms) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) - data2 = data2.map(input_columns=["image"], operations=transform()) + data2 = data2.map(input_columns=["image"], operations=transform) num_iter = 0 for item1, item2 in zip(data1.create_dict_iterator(num_epochs=1), data2.create_dict_iterator(num_epochs=1)): @@ -90,9 +91,9 @@ def test_type_cast_string(): transforms = [py_vision.Decode(), py_vision.ToTensor() ] - transform = py_vision.ComposeOp(transforms) + transform = mindspore.dataset.transforms.py_transforms.Compose(transforms) data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False) - data2 = data2.map(input_columns=["image"], operations=transform()) + data2 = data2.map(input_columns=["image"], operations=transform) num_iter = 0 for item1, item2 in zip(data1.create_dict_iterator(num_epochs=1), data2.create_dict_iterator(num_epochs=1)): diff --git a/tests/ut/python/dataset/test_uniform_augment.py b/tests/ut/python/dataset/test_uniform_augment.py index 10f896cf38..49464d924b 100644 --- a/tests/ut/python/dataset/test_uniform_augment.py +++ b/tests/ut/python/dataset/test_uniform_augment.py @@ -19,8 +19,9 @@ import numpy as np import pytest import mindspore.dataset.engine as de -import mindspore.dataset.transforms.vision.c_transforms as C -import mindspore.dataset.transforms.vision.py_transforms as F +import mindspore.dataset.transforms.py_transforms +import mindspore.dataset.vision.c_transforms as C +import mindspore.dataset.vision.py_transforms as F from mindspore import log as logger from util import visualize_list, diff_mse @@ -34,14 +35,14 @@ def test_uniform_augment(plot=False, num_ops=2): logger.info("Test UniformAugment") # Original Images - ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) - transforms_original = F.ComposeOp([F.Decode(), - F.Resize((224, 224)), - F.ToTensor()]) + transforms_original = mindspore.dataset.transforms.py_transforms.Compose([F.Decode(), + F.Resize((224, 224)), + F.ToTensor()]) ds_original = ds.map(input_columns="image", - operations=transforms_original()) + operations=transforms_original) ds_original = ds_original.batch(512) @@ -54,7 +55,7 @@ def test_uniform_augment(plot=False, num_ops=2): axis=0) # UniformAugment Images - ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) transform_list = [F.RandomRotation(45), F.RandomColor(), @@ -63,13 +64,15 @@ def test_uniform_augment(plot=False, num_ops=2): F.AutoContrast(), F.Equalize()] - transforms_ua = F.ComposeOp([F.Decode(), - F.Resize((224, 224)), - F.UniformAugment(transforms=transform_list, num_ops=num_ops), - F.ToTensor()]) + transforms_ua = \ + mindspore.dataset.transforms.py_transforms.Compose([F.Decode(), + F.Resize((224, 224)), + F.UniformAugment(transforms=transform_list, + num_ops=num_ops), + F.ToTensor()]) ds_ua = ds.map(input_columns="image", - operations=transforms_ua()) + operations=transforms_ua) ds_ua = ds_ua.batch(512) @@ -98,7 +101,7 @@ def test_cpp_uniform_augment(plot=False, num_ops=2): logger.info("Test CPP UniformAugment") # Original Images - ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) transforms_original = [C.Decode(), C.Resize(size=[224, 224]), F.ToTensor()] @@ -117,7 +120,7 @@ def test_cpp_uniform_augment(plot=False, num_ops=2): axis=0) # UniformAugment Images - ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False) + ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False) transforms_ua = [C.RandomCrop(size=[224, 224], padding=[32, 32, 32, 32]), C.RandomHorizontalFlip(), C.RandomVerticalFlip(), @@ -170,7 +173,7 @@ def test_cpp_uniform_augment_exception_pyops(num_ops=2): logger.info("Got an exception in DE: {}".format(str(e))) assert "Argument tensor_ops[5] with value" \ - " ,)" in str(e.value) diff --git a/tests/ut/python/dataset/test_var_batch_map.py b/tests/ut/python/dataset/test_var_batch_map.py index 5d9c0b709e..83c663437e 100644 --- a/tests/ut/python/dataset/test_var_batch_map.py +++ b/tests/ut/python/dataset/test_var_batch_map.py @@ -273,7 +273,7 @@ def test_var_batch_var_resize(): def add_one(batchInfo): return batchInfo.get_batch_num() + 1 - data1 = ds.ImageFolderDatasetV2("../data/dataset/testPK/data/", num_parallel_workers=4, decode=True) + data1 = ds.ImageFolderDataset("../data/dataset/testPK/data/", num_parallel_workers=4, decode=True) data1 = data1.batch(batch_size=add_one, drop_remainder=True, input_columns=["image"], per_batch_map=np_psedo_resize) # i-th batch has shape [i, i^2, i^2, 3] i = 1 diff --git a/tests/ut/python/dataset/util.py b/tests/ut/python/dataset/util.py index 667350d5e5..64f3fb129c 100644 --- a/tests/ut/python/dataset/util.py +++ b/tests/ut/python/dataset/util.py @@ -386,12 +386,12 @@ def check_bad_bbox(data, test_op, invalid_bbox_type, expected_error): # map to use selected invalid bounding box type data = data.map(input_columns=["image", "bbox"], output_columns=["image", "bbox"], - columns_order=["image", "bbox"], + column_order=["image", "bbox"], operations=lambda img, bboxes: add_bad_bbox(img, bboxes, invalid_bbox_type)) # map to apply ops data = data.map(input_columns=["image", "bbox"], output_columns=["image", "bbox"], - columns_order=["image", "bbox"], + column_order=["image", "bbox"], operations=[test_op]) # Add column for "bbox" for _, _ in enumerate(data.create_dict_iterator(num_epochs=1)): break