From 5d1ada682c5452c65315aa2174e13f6b68d066a4 Mon Sep 17 00:00:00 2001 From: Zirui Wu Date: Mon, 2 Nov 2020 16:44:27 -0500 Subject: [PATCH] remove uncessary headers staging II clean up further clean up fix merge conflict with 8139 fix complie err --- .../dataset/engine/datasetops/concat_op.cc | 16 ++++----- .../engine/ir/datasetops/source/csv_node.h | 2 +- .../ccsrc/minddata/dataset/include/datasets.h | 5 +-- mindspore/dataset/engine/datasets.py | 1 + .../lite/minddata/example/jni-example.cc | 1 - .../lite/minddata/example/x86-example.cc | 4 --- tests/ut/cpp/dataset/c_api_cache_test.cc | 14 -------- .../cpp/dataset/c_api_dataset_album_test.cc | 14 +------- .../cpp/dataset/c_api_dataset_cifar_test.cc | 18 ++-------- .../ut/cpp/dataset/c_api_dataset_clue_test.cc | 18 ++-------- .../ut/cpp/dataset/c_api_dataset_coco_test.cc | 33 ++----------------- .../cpp/dataset/c_api_dataset_config_test.cc | 13 ++------ .../ut/cpp/dataset/c_api_dataset_csv_test.cc | 30 +---------------- .../dataset/c_api_dataset_iterator_test.cc | 18 ++-------- .../dataset/c_api_dataset_manifest_test.cc | 14 +------- .../dataset/c_api_dataset_minddata_test.cc | 15 +-------- .../ut/cpp/dataset/c_api_dataset_ops_test.cc | 18 +--------- .../dataset/c_api_dataset_randomdata_test.cc | 14 -------- tests/ut/cpp/dataset/c_api_dataset_save.cc | 17 +--------- .../dataset/c_api_dataset_textfile_test.cc | 14 -------- .../dataset/c_api_dataset_tfrecord_test.cc | 14 -------- .../ut/cpp/dataset/c_api_dataset_voc_test.cc | 17 ++-------- tests/ut/cpp/dataset/c_api_datasets_test.cc | 19 ++--------- tests/ut/cpp/dataset/c_api_samplers_test.cc | 9 +---- .../c_api_text_sentence_piece_vocab_test.cc | 10 +++--- tests/ut/cpp/dataset/c_api_text_vocab_test.cc | 19 +++-------- tests/ut/cpp/dataset/c_api_transforms_test.cc | 32 ++---------------- tests/ut/cpp/dataset/c_api_vision_test.cc | 20 ++--------- 28 files changed, 52 insertions(+), 367 deletions(-) diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/concat_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/concat_op.cc index a31bcdc2a5..816945d744 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/concat_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/concat_op.cc @@ -13,16 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +#include "minddata/dataset/engine/datasetops/concat_op.h" + #include #include -#include "utils/ms_utils.h" #include "minddata/dataset/core/config_manager.h" #include "minddata/dataset/engine/data_buffer.h" -#include "minddata/dataset/engine/datasetops/concat_op.h" -#include "minddata/dataset/engine/opt/pass.h" #include "minddata/dataset/engine/db_connector.h" -#include "minddata/dataset/engine/execution_tree.h" +#include "minddata/dataset/engine/opt/pass.h" +#include "utils/ms_utils.h" namespace mindspore { namespace dataset { @@ -101,7 +101,7 @@ Status ConcatOp::operator()() { // 3. Put the data into output_connector if (!children_flag_and_nums_.empty()) is_not_mappable = children_flag_and_nums_[i].first; while (!buf->eoe() && !buf->eof()) { - // if dataset is no mappable or generator dataset which source is yeild（cannot get the number of samples in + // if dataset is not mappable or generator dataset which source is yield, cannot get the number of samples in // python layer), we use filtering to get data if (sample_number % num_shard == shard_index && (is_not_mappable || !children_flag_and_nums_[i].second)) { RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(buf))); @@ -125,7 +125,7 @@ Status ConcatOp::operator()() { RETURN_IF_NOT_OK(child_[i]->GetNextBuffer(&buf)); } - // if dataset is mappable,We do't use filtering to pick data. + // if dataset is mappable,We don't use filtering to pick data. // so sample_number plus the length of the entire dataset if (!is_not_mappable && children_flag_and_nums_[i].second) { sample_number += children_flag_and_nums_[i].second; @@ -142,7 +142,7 @@ Status ConcatOp::operator()() { CHECK_FAIL_RETURN_UNEXPECTED(eof_count == children_num_, "Something went wrong, eof count does not match the number of children."); // 5. Add eof buffer in the end manually - MS_LOG(DEBUG) << "Add the eof buffer manualy in the end."; + MS_LOG(DEBUG) << "Add the eof buffer manually in the end."; auto eof_buffer = std::make_unique(0, DataBuffer::kDeBFlagEOF); RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(eof_buffer))); return Status::OK(); @@ -150,7 +150,7 @@ Status ConcatOp::operator()() { Status ConcatOp::Verify(int32_t id, const std::unique_ptr &buf) { TensorRow new_row; - buf->GetRow(0, &new_row); + RETURN_IF_NOT_OK(buf->GetRow(0, &new_row)); if (id == 0) { // Obtain the data type and data rank in child[0] diff --git a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/csv_node.h b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/csv_node.h index e006b237e4..bb7dae493f 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/csv_node.h +++ b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/csv_node.h @@ -25,10 +25,10 @@ namespace mindspore { namespace dataset { -/// \brief Base class of CSV Record /// \brief Record type for CSV enum CsvType : uint8_t { INT = 0, FLOAT, STRING }; +/// \brief Base class of CSV Record class CsvBase { public: CsvBase() = default; diff --git a/mindspore/ccsrc/minddata/dataset/include/datasets.h b/mindspore/ccsrc/minddata/dataset/include/datasets.h index 9858388580..3760e7647a 100644 --- a/mindspore/ccsrc/minddata/dataset/include/datasets.h +++ b/mindspore/ccsrc/minddata/dataset/include/datasets.h @@ -61,13 +61,14 @@ class Iterator; class TensorOperation; class SchemaObj; class SamplerObj; -// Datasets classes (in alphabetical order) class CsvBase; + +// Dataset classes (in alphabetical order) class BatchDataset; -// Dataset Op classes (in alphabetical order) #ifndef ENABLE_ANDROID class BucketBatchByLengthDataset; #endif +class CSVDataset; class ConcatDataset; class MapDataset; class ProjectDataset; diff --git a/mindspore/dataset/engine/datasets.py b/mindspore/dataset/engine/datasets.py index a8a2bb785e..8ed9647638 100644 --- a/mindspore/dataset/engine/datasets.py +++ b/mindspore/dataset/engine/datasets.py @@ -2521,6 +2521,7 @@ class ConcatDataset(DatasetOp): if isinstance(child, BatchDataset): raise TypeError("The parameter %s of concat must not be BatchDataset!" % (child)) + # if child is mappable and the length is greater than 0 if not self._children_flag_and_nums[index][0] and self._children_flag_and_nums[index][1]: tem_value = cumulative_samples_nums + self._children_flag_and_nums[index][1] diff --git a/mindspore/lite/minddata/example/jni-example.cc b/mindspore/lite/minddata/example/jni-example.cc index 0b46470c7d..ed6b58725b 100644 --- a/mindspore/lite/minddata/example/jni-example.cc +++ b/mindspore/lite/minddata/example/jni-example.cc @@ -20,7 +20,6 @@ #include #include #include "minddata/dataset/include/datasets.h" -#include "minddata/dataset/engine/ir/datasetops/source/cifar10_node.h" #include "minddata/dataset/util/path.h" #if defined(__ANDROID__) || defined(ANDROID) #include diff --git a/mindspore/lite/minddata/example/x86-example.cc b/mindspore/lite/minddata/example/x86-example.cc index e0ac1f9f69..9b51789daa 100644 --- a/mindspore/lite/minddata/example/x86-example.cc +++ b/mindspore/lite/minddata/example/x86-example.cc @@ -15,12 +15,8 @@ */ #include -#include -#include #include #include "minddata/dataset/include/datasets.h" -#include "minddata/dataset/engine/ir/datasetops/source/cifar10_node.h" -#include "minddata/dataset/util/path.h" using Dataset = mindspore::dataset::Dataset; using Iterator = mindspore::dataset::Iterator; diff --git a/tests/ut/cpp/dataset/c_api_cache_test.cc b/tests/ut/cpp/dataset/c_api_cache_test.cc index 0da852f335..9b4eb6348e 100644 --- a/tests/ut/cpp/dataset/c_api_cache_test.cc +++ b/tests/ut/cpp/dataset/c_api_cache_test.cc @@ -16,21 +16,7 @@ #include "common/common.h" #include "minddata/dataset/include/datasets.h" -// IR leaf nodes -#include "minddata/dataset/engine/ir/datasetops/source/album_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/celeba_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/cifar10_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/cifar100_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/clue_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/coco_node.h" #include "minddata/dataset/engine/ir/datasetops/source/csv_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/image_folder_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/manifest_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/mnist_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/random_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/text_file_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/tf_record_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/voc_node.h" using namespace mindspore::dataset; diff --git a/tests/ut/cpp/dataset/c_api_dataset_album_test.cc b/tests/ut/cpp/dataset/c_api_dataset_album_test.cc index 5e3dec81c0..a4e1c26a7d 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_album_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_album_test.cc @@ -16,20 +16,8 @@ #include "common/common.h" #include "minddata/dataset/include/datasets.h" -// IR non-leaf nodes -#include "minddata/dataset/engine/ir/datasetops/batch_node.h" -#include "minddata/dataset/engine/ir/datasetops/bucket_batch_by_length_node.h" -#include "minddata/dataset/engine/ir/datasetops/concat_node.h" -#include "minddata/dataset/engine/ir/datasetops/map_node.h" -#include "minddata/dataset/engine/ir/datasetops/project_node.h" -#include "minddata/dataset/engine/ir/datasetops/rename_node.h" -#include "minddata/dataset/engine/ir/datasetops/shuffle_node.h" -#include "minddata/dataset/engine/ir/datasetops/zip_node.h" - -// IR leaf nodes -#include "minddata/dataset/engine/ir/datasetops/source/album_node.h" - using namespace mindspore::dataset; +using mindspore::dataset::Tensor; class MindDataTestPipeline : public UT::DatasetOpTesting { protected: diff --git a/tests/ut/cpp/dataset/c_api_dataset_cifar_test.cc b/tests/ut/cpp/dataset/c_api_dataset_cifar_test.cc index 9c6fd74944..1c586d7f63 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_cifar_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_cifar_test.cc @@ -16,22 +16,10 @@ #include "common/common.h" #include "minddata/dataset/include/datasets.h" -// IR non-leaf nodes -#include "minddata/dataset/engine/ir/datasetops/batch_node.h" -#include "minddata/dataset/engine/ir/datasetops/bucket_batch_by_length_node.h" -#include "minddata/dataset/engine/ir/datasetops/concat_node.h" -#include "minddata/dataset/engine/ir/datasetops/map_node.h" -#include "minddata/dataset/engine/ir/datasetops/project_node.h" -#include "minddata/dataset/engine/ir/datasetops/rename_node.h" -#include "minddata/dataset/engine/ir/datasetops/shuffle_node.h" -#include "minddata/dataset/engine/ir/datasetops/skip_node.h" -#include "minddata/dataset/engine/ir/datasetops/zip_node.h" - -// IR leaf nodes -#include "minddata/dataset/engine/ir/datasetops/source/cifar100_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/cifar10_node.h" - using namespace mindspore::dataset; +using mindspore::dataset::DataType; +using mindspore::dataset::Tensor; +using mindspore::dataset::TensorShape; class MindDataTestPipeline : public UT::DatasetOpTesting { protected: diff --git a/tests/ut/cpp/dataset/c_api_dataset_clue_test.cc b/tests/ut/cpp/dataset/c_api_dataset_clue_test.cc index c3758cfc7d..bb9ead4d3f 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_clue_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_clue_test.cc @@ -14,25 +14,13 @@ * limitations under the License. */ #include "common/common.h" -#include "minddata/dataset/core/config_manager.h" #include "minddata/dataset/core/global_context.h" #include "minddata/dataset/include/datasets.h" -// IR non-leaf nodes -#include "minddata/dataset/engine/ir/datasetops/batch_node.h" -#include "minddata/dataset/engine/ir/datasetops/bucket_batch_by_length_node.h" -#include "minddata/dataset/engine/ir/datasetops/concat_node.h" -#include "minddata/dataset/engine/ir/datasetops/map_node.h" -#include "minddata/dataset/engine/ir/datasetops/project_node.h" -#include "minddata/dataset/engine/ir/datasetops/rename_node.h" -#include "minddata/dataset/engine/ir/datasetops/shuffle_node.h" -#include "minddata/dataset/engine/ir/datasetops/skip_node.h" -#include "minddata/dataset/engine/ir/datasetops/zip_node.h" - -// IR leaf nodes -#include "minddata/dataset/engine/ir/datasetops/source/clue_node.h" - using namespace mindspore::dataset; +using mindspore::dataset::GlobalContext; +using mindspore::dataset::ShuffleMode; +using mindspore::dataset::Tensor; class MindDataTestPipeline : public UT::DatasetOpTesting { protected: diff --git a/tests/ut/cpp/dataset/c_api_dataset_coco_test.cc b/tests/ut/cpp/dataset/c_api_dataset_coco_test.cc index ae64483616..48e3ee2e08 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_coco_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_coco_test.cc @@ -16,37 +16,10 @@ #include "common/common.h" #include "minddata/dataset/include/datasets.h" -// IR non-leaf nodes -#include "minddata/dataset/engine/ir/datasetops/batch_node.h" -#include "minddata/dataset/engine/ir/datasetops/bucket_batch_by_length_node.h" -#include "minddata/dataset/engine/ir/datasetops/build_vocab_node.h" -#include "minddata/dataset/engine/ir/datasetops/concat_node.h" -#include "minddata/dataset/engine/ir/datasetops/map_node.h" -#include "minddata/dataset/engine/ir/datasetops/project_node.h" -#include "minddata/dataset/engine/ir/datasetops/rename_node.h" -#include "minddata/dataset/engine/ir/datasetops/repeat_node.h" -#include "minddata/dataset/engine/ir/datasetops/shuffle_node.h" -#include "minddata/dataset/engine/ir/datasetops/skip_node.h" -#include "minddata/dataset/engine/ir/datasetops/take_node.h" -#include "minddata/dataset/engine/ir/datasetops/zip_node.h" - -// IR leaf nodes -#include "minddata/dataset/engine/ir/datasetops/source/album_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/celeba_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/cifar100_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/cifar10_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/clue_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/coco_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/csv_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/image_folder_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/manifest_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/minddata_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/random_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/text_file_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/tf_record_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/voc_node.h" - using namespace mindspore::dataset; +using mindspore::dataset::dsize_t; +using mindspore::dataset::Tensor; +using mindspore::dataset::TensorShape; class MindDataTestPipeline : public UT::DatasetOpTesting { protected: diff --git a/tests/ut/cpp/dataset/c_api_dataset_config_test.cc b/tests/ut/cpp/dataset/c_api_dataset_config_test.cc index 54a3e4e565..9606fe6c7c 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_config_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_config_test.cc @@ -18,18 +18,9 @@ #include "minddata/dataset/include/config.h" #include "minddata/dataset/include/datasets.h" -// IR non-leaf nodes -#include "minddata/dataset/engine/ir/datasetops/batch_node.h" -#include "minddata/dataset/engine/ir/datasetops/concat_node.h" -#include "minddata/dataset/engine/ir/datasetops/map_node.h" -#include "minddata/dataset/engine/ir/datasetops/project_node.h" -#include "minddata/dataset/engine/ir/datasetops/rename_node.h" -#include "minddata/dataset/engine/ir/datasetops/shuffle_node.h" - -// IR leaf nodes -#include "minddata/dataset/engine/ir/datasetops/source/text_file_node.h" - using namespace mindspore::dataset; +using mindspore::dataset::ShuffleMode; +using mindspore::dataset::Tensor; class MindDataTestPipeline : public UT::DatasetOpTesting { protected: diff --git a/tests/ut/cpp/dataset/c_api_dataset_csv_test.cc b/tests/ut/cpp/dataset/c_api_dataset_csv_test.cc index 98aada926b..af6fc24e53 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_csv_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_csv_test.cc @@ -14,39 +14,11 @@ * limitations under the License. */ #include "common/common.h" -#include "minddata/dataset/core/config_manager.h" #include "minddata/dataset/core/global_context.h" #include "minddata/dataset/include/datasets.h" -// IR non-leaf nodes -#include "minddata/dataset/engine/ir/datasetops/batch_node.h" -#include "minddata/dataset/engine/ir/datasetops/bucket_batch_by_length_node.h" -#include "minddata/dataset/engine/ir/datasetops/build_vocab_node.h" -#include "minddata/dataset/engine/ir/datasetops/concat_node.h" -#include "minddata/dataset/engine/ir/datasetops/map_node.h" -#include "minddata/dataset/engine/ir/datasetops/project_node.h" -#include "minddata/dataset/engine/ir/datasetops/rename_node.h" -#include "minddata/dataset/engine/ir/datasetops/repeat_node.h" -#include "minddata/dataset/engine/ir/datasetops/shuffle_node.h" -#include "minddata/dataset/engine/ir/datasetops/skip_node.h" -#include "minddata/dataset/engine/ir/datasetops/take_node.h" -#include "minddata/dataset/engine/ir/datasetops/zip_node.h" - -// IR leaf nodes -#include "minddata/dataset/engine/ir/datasetops/source/album_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/celeba_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/cifar100_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/cifar10_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/clue_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/coco_node.h" +// need for CsvRecord #include "minddata/dataset/engine/ir/datasetops/source/csv_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/image_folder_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/manifest_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/minddata_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/random_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/text_file_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/tf_record_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/voc_node.h" using namespace mindspore::dataset; diff --git a/tests/ut/cpp/dataset/c_api_dataset_iterator_test.cc b/tests/ut/cpp/dataset/c_api_dataset_iterator_test.cc index 3d3f503cbe..0250d39da0 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_iterator_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_iterator_test.cc @@ -17,23 +17,9 @@ #include "common/common.h" #include "minddata/dataset/include/datasets.h" -// IR non-leaf nodes -#include "minddata/dataset/engine/ir/datasetops/batch_node.h" -#include "minddata/dataset/engine/ir/datasetops/bucket_batch_by_length_node.h" -#include "minddata/dataset/engine/ir/datasetops/concat_node.h" -#include "minddata/dataset/engine/ir/datasetops/map_node.h" -#include "minddata/dataset/engine/ir/datasetops/project_node.h" -#include "minddata/dataset/engine/ir/datasetops/rename_node.h" -#include "minddata/dataset/engine/ir/datasetops/shuffle_node.h" -#include "minddata/dataset/engine/ir/datasetops/skip_node.h" -#include "minddata/dataset/engine/ir/datasetops/zip_node.h" - -// IR leaf nodes -#include "minddata/dataset/engine/ir/datasetops/source/cifar10_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/mnist_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/voc_node.h" - using namespace mindspore::dataset; +using mindspore::dataset::Tensor; +using mindspore::dataset::TensorShape; class MindDataTestPipeline : public UT::DatasetOpTesting { protected: diff --git a/tests/ut/cpp/dataset/c_api_dataset_manifest_test.cc b/tests/ut/cpp/dataset/c_api_dataset_manifest_test.cc index bc67747d89..815ec6c9c5 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_manifest_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_manifest_test.cc @@ -16,20 +16,8 @@ #include "common/common.h" #include "minddata/dataset/include/datasets.h" -// IR non-leaf nodes -#include "minddata/dataset/engine/ir/datasetops/batch_node.h" -#include "minddata/dataset/engine/ir/datasetops/bucket_batch_by_length_node.h" -#include "minddata/dataset/engine/ir/datasetops/concat_node.h" -#include "minddata/dataset/engine/ir/datasetops/map_node.h" -#include "minddata/dataset/engine/ir/datasetops/project_node.h" -#include "minddata/dataset/engine/ir/datasetops/rename_node.h" -#include "minddata/dataset/engine/ir/datasetops/shuffle_node.h" -#include "minddata/dataset/engine/ir/datasetops/zip_node.h" - -// IR leaf nodes -#include "minddata/dataset/engine/ir/datasetops/source/manifest_node.h" - using namespace mindspore::dataset; +using mindspore::dataset::Tensor; class MindDataTestPipeline : public UT::DatasetOpTesting { protected: diff --git a/tests/ut/cpp/dataset/c_api_dataset_minddata_test.cc b/tests/ut/cpp/dataset/c_api_dataset_minddata_test.cc index b7bde3da13..eca50b951f 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_minddata_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_minddata_test.cc @@ -16,21 +16,8 @@ #include "common/common.h" #include "minddata/dataset/include/datasets.h" -// IR non-leaf nodes -#include "minddata/dataset/engine/ir/datasetops/batch_node.h" -#include "minddata/dataset/engine/ir/datasetops/bucket_batch_by_length_node.h" -#include "minddata/dataset/engine/ir/datasetops/concat_node.h" -#include "minddata/dataset/engine/ir/datasetops/map_node.h" -#include "minddata/dataset/engine/ir/datasetops/project_node.h" -#include "minddata/dataset/engine/ir/datasetops/rename_node.h" -#include "minddata/dataset/engine/ir/datasetops/shuffle_node.h" -#include "minddata/dataset/engine/ir/datasetops/skip_node.h" -#include "minddata/dataset/engine/ir/datasetops/zip_node.h" - -// IR leaf nodes -#include "minddata/dataset/engine/ir/datasetops/source/minddata_node.h" - using namespace mindspore::dataset; +using mindspore::dataset::Tensor; class MindDataTestPipeline : public UT::DatasetOpTesting { protected: diff --git a/tests/ut/cpp/dataset/c_api_dataset_ops_test.cc b/tests/ut/cpp/dataset/c_api_dataset_ops_test.cc index 74aa2b0cb6..ff74363e30 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_ops_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_ops_test.cc @@ -18,24 +18,8 @@ #include "minddata/dataset/include/datasets.h" #include "minddata/dataset/include/vision.h" -// IR non-leaf nodes -#include "minddata/dataset/engine/ir/datasetops/batch_node.h" -#include "minddata/dataset/engine/ir/datasetops/bucket_batch_by_length_node.h" -#include "minddata/dataset/engine/ir/datasetops/concat_node.h" -#include "minddata/dataset/engine/ir/datasetops/map_node.h" -#include "minddata/dataset/engine/ir/datasetops/project_node.h" -#include "minddata/dataset/engine/ir/datasetops/rename_node.h" -#include "minddata/dataset/engine/ir/datasetops/shuffle_node.h" -#include "minddata/dataset/engine/ir/datasetops/skip_node.h" -#include "minddata/dataset/engine/ir/datasetops/zip_node.h" - -// IR leaf nodes - -#include "minddata/dataset/engine/ir/datasetops/source/cifar10_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/image_folder_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/mnist_node.h" - using namespace mindspore::dataset; +using mindspore::dataset::Tensor; class MindDataTestPipeline : public UT::DatasetOpTesting { protected: diff --git a/tests/ut/cpp/dataset/c_api_dataset_randomdata_test.cc b/tests/ut/cpp/dataset/c_api_dataset_randomdata_test.cc index 42429097e5..59d017ec5c 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_randomdata_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_randomdata_test.cc @@ -19,20 +19,6 @@ #include "mindspore/core/ir/dtype/type_id.h" -// IR non-leaf nodes -#include "minddata/dataset/engine/ir/datasetops/batch_node.h" -#include "minddata/dataset/engine/ir/datasetops/bucket_batch_by_length_node.h" -#include "minddata/dataset/engine/ir/datasetops/concat_node.h" -#include "minddata/dataset/engine/ir/datasetops/map_node.h" -#include "minddata/dataset/engine/ir/datasetops/project_node.h" -#include "minddata/dataset/engine/ir/datasetops/rename_node.h" -#include "minddata/dataset/engine/ir/datasetops/shuffle_node.h" -#include "minddata/dataset/engine/ir/datasetops/skip_node.h" -#include "minddata/dataset/engine/ir/datasetops/zip_node.h" - -// IR leaf nodes -#include "minddata/dataset/engine/ir/datasetops/source/random_node.h" - using namespace mindspore::dataset; class MindDataTestPipeline : public UT::DatasetOpTesting { diff --git a/tests/ut/cpp/dataset/c_api_dataset_save.cc b/tests/ut/cpp/dataset/c_api_dataset_save.cc index 61bbb1930a..88b3786ecd 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_save.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_save.cc @@ -18,23 +18,8 @@ #include "minddata/dataset/include/datasets.h" #include "minddata/dataset/include/transforms.h" -// IR non-leaf nodes -#include "minddata/dataset/engine/ir/datasetops/batch_node.h" -#include "minddata/dataset/engine/ir/datasetops/bucket_batch_by_length_node.h" -#include "minddata/dataset/engine/ir/datasetops/concat_node.h" -#include "minddata/dataset/engine/ir/datasetops/map_node.h" -#include "minddata/dataset/engine/ir/datasetops/project_node.h" -#include "minddata/dataset/engine/ir/datasetops/rename_node.h" -#include "minddata/dataset/engine/ir/datasetops/shuffle_node.h" -#include "minddata/dataset/engine/ir/datasetops/skip_node.h" -#include "minddata/dataset/engine/ir/datasetops/zip_node.h" - -// IR leaf nodes - -#include "minddata/dataset/engine/ir/datasetops/source/cifar10_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/minddata_node.h" - using namespace mindspore::dataset; +using mindspore::dataset::Tensor; class MindDataTestPipeline : public UT::DatasetOpTesting { protected: diff --git a/tests/ut/cpp/dataset/c_api_dataset_textfile_test.cc b/tests/ut/cpp/dataset/c_api_dataset_textfile_test.cc index 3da9e45708..1735274fba 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_textfile_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_textfile_test.cc @@ -17,20 +17,6 @@ #include "minddata/dataset/core/global_context.h" #include "minddata/dataset/include/datasets.h" -// IR non-leaf nodes -#include "minddata/dataset/engine/ir/datasetops/batch_node.h" -#include "minddata/dataset/engine/ir/datasetops/bucket_batch_by_length_node.h" -#include "minddata/dataset/engine/ir/datasetops/concat_node.h" -#include "minddata/dataset/engine/ir/datasetops/map_node.h" -#include "minddata/dataset/engine/ir/datasetops/project_node.h" -#include "minddata/dataset/engine/ir/datasetops/rename_node.h" -#include "minddata/dataset/engine/ir/datasetops/shuffle_node.h" -#include "minddata/dataset/engine/ir/datasetops/skip_node.h" -#include "minddata/dataset/engine/ir/datasetops/zip_node.h" - -// IR leaf nodes -#include "minddata/dataset/engine/ir/datasetops/source/text_file_node.h" - using namespace mindspore::dataset; using mindspore::dataset::ShuffleMode; diff --git a/tests/ut/cpp/dataset/c_api_dataset_tfrecord_test.cc b/tests/ut/cpp/dataset/c_api_dataset_tfrecord_test.cc index a590bc3270..59841f6515 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_tfrecord_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_tfrecord_test.cc @@ -18,20 +18,6 @@ #include "minddata/dataset/include/vision.h" #include "minddata/dataset/core/global_context.h" -// IR non-leaf nodes -#include "minddata/dataset/engine/ir/datasetops/batch_node.h" -#include "minddata/dataset/engine/ir/datasetops/bucket_batch_by_length_node.h" -#include "minddata/dataset/engine/ir/datasetops/concat_node.h" -#include "minddata/dataset/engine/ir/datasetops/map_node.h" -#include "minddata/dataset/engine/ir/datasetops/project_node.h" -#include "minddata/dataset/engine/ir/datasetops/rename_node.h" -#include "minddata/dataset/engine/ir/datasetops/shuffle_node.h" -#include "minddata/dataset/engine/ir/datasetops/skip_node.h" -#include "minddata/dataset/engine/ir/datasetops/zip_node.h" - -// IR leaf nodes -#include "minddata/dataset/engine/ir/datasetops/source/tf_record_node.h" - using namespace mindspore::dataset; using mindspore::dataset::DataType; diff --git a/tests/ut/cpp/dataset/c_api_dataset_voc_test.cc b/tests/ut/cpp/dataset/c_api_dataset_voc_test.cc index b15058beeb..fe2e274921 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_voc_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_voc_test.cc @@ -16,21 +16,10 @@ #include "common/common.h" #include "minddata/dataset/include/datasets.h" -// IR non-leaf nodes -#include "minddata/dataset/engine/ir/datasetops/batch_node.h" -#include "minddata/dataset/engine/ir/datasetops/bucket_batch_by_length_node.h" -#include "minddata/dataset/engine/ir/datasetops/concat_node.h" -#include "minddata/dataset/engine/ir/datasetops/map_node.h" -#include "minddata/dataset/engine/ir/datasetops/project_node.h" -#include "minddata/dataset/engine/ir/datasetops/rename_node.h" -#include "minddata/dataset/engine/ir/datasetops/shuffle_node.h" -#include "minddata/dataset/engine/ir/datasetops/skip_node.h" -#include "minddata/dataset/engine/ir/datasetops/zip_node.h" - -// IR leaf nodes -#include "minddata/dataset/engine/ir/datasetops/source/voc_node.h" - using namespace mindspore::dataset; +using mindspore::dataset::DataType; +using mindspore::dataset::Tensor; +using mindspore::dataset::TensorShape; class MindDataTestPipeline : public UT::DatasetOpTesting { protected: diff --git a/tests/ut/cpp/dataset/c_api_datasets_test.cc b/tests/ut/cpp/dataset/c_api_datasets_test.cc index 6079d2b977..080a60142a 100644 --- a/tests/ut/cpp/dataset/c_api_datasets_test.cc +++ b/tests/ut/cpp/dataset/c_api_datasets_test.cc @@ -16,24 +16,9 @@ #include "common/common.h" #include "minddata/dataset/include/datasets.h" -// IR non-leaf nodes -#include "minddata/dataset/engine/ir/datasetops/batch_node.h" -#include "minddata/dataset/engine/ir/datasetops/bucket_batch_by_length_node.h" -#include "minddata/dataset/engine/ir/datasetops/concat_node.h" -#include "minddata/dataset/engine/ir/datasetops/map_node.h" -#include "minddata/dataset/engine/ir/datasetops/project_node.h" -#include "minddata/dataset/engine/ir/datasetops/rename_node.h" -#include "minddata/dataset/engine/ir/datasetops/shuffle_node.h" -#include "minddata/dataset/engine/ir/datasetops/skip_node.h" -#include "minddata/dataset/engine/ir/datasetops/zip_node.h" - -// IR leaf nodes - -#include "minddata/dataset/engine/ir/datasetops/source/celeba_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/image_folder_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/mnist_node.h" - using namespace mindspore::dataset; +using mindspore::dataset::Tensor; +using mindspore::dataset::TensorShape; class MindDataTestPipeline : public UT::DatasetOpTesting { protected: diff --git a/tests/ut/cpp/dataset/c_api_samplers_test.cc b/tests/ut/cpp/dataset/c_api_samplers_test.cc index 47a7af349a..b4931d0e3d 100644 --- a/tests/ut/cpp/dataset/c_api_samplers_test.cc +++ b/tests/ut/cpp/dataset/c_api_samplers_test.cc @@ -16,15 +16,8 @@ #include "common/common.h" #include "minddata/dataset/include/datasets.h" -#include "minddata/dataset/engine/ir/datasetops/batch_node.h" -#include "minddata/dataset/engine/ir/datasetops/bucket_batch_by_length_node.h" -#include "minddata/dataset/engine/ir/datasetops/concat_node.h" -#include "minddata/dataset/engine/ir/datasetops/project_node.h" -#include "minddata/dataset/engine/ir/datasetops/rename_node.h" -#include "minddata/dataset/engine/ir/datasetops/shuffle_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/image_folder_node.h" - using namespace mindspore::dataset; +using mindspore::dataset::Tensor; class MindDataTestPipeline : public UT::DatasetOpTesting { protected: diff --git a/tests/ut/cpp/dataset/c_api_text_sentence_piece_vocab_test.cc b/tests/ut/cpp/dataset/c_api_text_sentence_piece_vocab_test.cc index 2c89ae4a1f..ba638834b2 100644 --- a/tests/ut/cpp/dataset/c_api_text_sentence_piece_vocab_test.cc +++ b/tests/ut/cpp/dataset/c_api_text_sentence_piece_vocab_test.cc @@ -23,13 +23,11 @@ #include "minddata/dataset/include/transforms.h" #include "minddata/dataset/include/text.h" -// IR non-leaf nodes -#include "minddata/dataset/engine/ir/datasetops/map_node.h" - -// IR leaf nodes -#include "minddata/dataset/engine/ir/datasetops/source/text_file_node.h" - using namespace mindspore::dataset; +using mindspore::dataset::SentencePieceModel; +using mindspore::dataset::SentencePieceVocab; +using mindspore::dataset::ShuffleMode; +using mindspore::dataset::Tensor; class MindDataTestPipeline : public UT::DatasetOpTesting { protected: diff --git a/tests/ut/cpp/dataset/c_api_text_vocab_test.cc b/tests/ut/cpp/dataset/c_api_text_vocab_test.cc index 9dd7afb55f..008330b24a 100644 --- a/tests/ut/cpp/dataset/c_api_text_vocab_test.cc +++ b/tests/ut/cpp/dataset/c_api_text_vocab_test.cc @@ -23,21 +23,12 @@ #include "minddata/dataset/include/transforms.h" #include "minddata/dataset/include/text.h" -// IR non-leaf nodes -#include "minddata/dataset/engine/ir/datasetops/batch_node.h" -#include "minddata/dataset/engine/ir/datasetops/bucket_batch_by_length_node.h" -#include "minddata/dataset/engine/ir/datasetops/concat_node.h" -#include "minddata/dataset/engine/ir/datasetops/map_node.h" -#include "minddata/dataset/engine/ir/datasetops/project_node.h" -#include "minddata/dataset/engine/ir/datasetops/rename_node.h" -#include "minddata/dataset/engine/ir/datasetops/shuffle_node.h" -#include "minddata/dataset/engine/ir/datasetops/skip_node.h" -#include "minddata/dataset/engine/ir/datasetops/zip_node.h" - -// IR leaf nodes -#include "minddata/dataset/engine/ir/datasetops/source/text_file_node.h" - using namespace mindspore::dataset; +using mindspore::dataset::DataType; +using mindspore::dataset::ShuffleMode; +using mindspore::dataset::Status; +using mindspore::dataset::Tensor; +using mindspore::dataset::Vocab; class MindDataTestPipeline : public UT::DatasetOpTesting { protected: diff --git a/tests/ut/cpp/dataset/c_api_transforms_test.cc b/tests/ut/cpp/dataset/c_api_transforms_test.cc index 86b860549d..c0e96eeca6 100644 --- a/tests/ut/cpp/dataset/c_api_transforms_test.cc +++ b/tests/ut/cpp/dataset/c_api_transforms_test.cc @@ -18,37 +18,9 @@ #include "minddata/dataset/include/transforms.h" #include "minddata/dataset/include/vision.h" -// IR non-leaf nodes -#include "minddata/dataset/engine/ir/datasetops/batch_node.h" -#include "minddata/dataset/engine/ir/datasetops/bucket_batch_by_length_node.h" -#include "minddata/dataset/engine/ir/datasetops/build_vocab_node.h" -#include "minddata/dataset/engine/ir/datasetops/concat_node.h" -#include "minddata/dataset/engine/ir/datasetops/map_node.h" -#include "minddata/dataset/engine/ir/datasetops/project_node.h" -#include "minddata/dataset/engine/ir/datasetops/rename_node.h" -#include "minddata/dataset/engine/ir/datasetops/repeat_node.h" -#include "minddata/dataset/engine/ir/datasetops/shuffle_node.h" -#include "minddata/dataset/engine/ir/datasetops/skip_node.h" -#include "minddata/dataset/engine/ir/datasetops/take_node.h" -#include "minddata/dataset/engine/ir/datasetops/zip_node.h" - -// IR leaf nodes -#include "minddata/dataset/engine/ir/datasetops/source/album_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/celeba_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/cifar100_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/cifar10_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/clue_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/csv_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/csv_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/image_folder_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/manifest_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/minddata_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/random_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/text_file_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/tf_record_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/voc_node.h" - using namespace mindspore::dataset; +using mindspore::dataset::BorderType; +using mindspore::dataset::Tensor; class MindDataTestPipeline : public UT::DatasetOpTesting { protected: diff --git a/tests/ut/cpp/dataset/c_api_vision_test.cc b/tests/ut/cpp/dataset/c_api_vision_test.cc index 4420c67152..ca37dd2204 100644 --- a/tests/ut/cpp/dataset/c_api_vision_test.cc +++ b/tests/ut/cpp/dataset/c_api_vision_test.cc @@ -18,24 +18,10 @@ #include "minddata/dataset/include/transforms.h" #include "minddata/dataset/include/vision.h" -// IR non-leaf nodes -#include "minddata/dataset/engine/ir/datasetops/batch_node.h" -#include "minddata/dataset/engine/ir/datasetops/bucket_batch_by_length_node.h" -#include "minddata/dataset/engine/ir/datasetops/concat_node.h" -#include "minddata/dataset/engine/ir/datasetops/map_node.h" -#include "minddata/dataset/engine/ir/datasetops/project_node.h" -#include "minddata/dataset/engine/ir/datasetops/rename_node.h" -#include "minddata/dataset/engine/ir/datasetops/shuffle_node.h" -#include "minddata/dataset/engine/ir/datasetops/skip_node.h" -#include "minddata/dataset/engine/ir/datasetops/zip_node.h" - -// IR leaf nodes -#include "minddata/dataset/engine/ir/datasetops/source/cifar10_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/image_folder_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/mnist_node.h" -#include "minddata/dataset/engine/ir/datasetops/source/voc_node.h" - using namespace mindspore::dataset; +using mindspore::dataset::BorderType; +using mindspore::dataset::InterpolationMode; +using mindspore::dataset::Tensor; class MindDataTestPipeline : public UT::DatasetOpTesting { protected: