From 518828b639512222a57b3cc38f5384f9991192c9 Mon Sep 17 00:00:00 2001 From: TinaMengtingZhang Date: Fri, 19 Feb 2021 18:18:33 -0500 Subject: [PATCH] Support 3 composed data ops and cleanup code --- .../ccsrc/minddata/dataset/api/datasets.cc | 1 - .../ccsrc/minddata/dataset/api/execute.cc | 1 + .../ccsrc/minddata/dataset/api/transforms.cc | 89 +++++-- .../dataset/engine/ir/datasetops/map_node.cc | 2 +- .../engine/opt/pre/cache_validation_pass.cc | 1 + .../ccsrc/minddata/dataset/include/datasets.h | 17 +- .../ccsrc/minddata/dataset/include/text.h | 32 +-- .../minddata/dataset/include/transforms.h | 94 +++++-- .../minddata/dataset/kernels/ir/validators.cc | 2 + .../minddata/dataset/kernels/ir/validators.h | 3 +- .../dataset/c_api_dataset_iterator_test.cc | 2 +- .../dataset/c_api_dataset_manifest_test.cc | 2 +- tests/ut/cpp/dataset/c_api_transforms_test.cc | 248 ++++++++++++++---- .../ut/cpp/dataset/optimization_pass_test.cc | 27 +- 14 files changed, 387 insertions(+), 134 deletions(-) diff --git a/mindspore/ccsrc/minddata/dataset/api/datasets.cc b/mindspore/ccsrc/minddata/dataset/api/datasets.cc index 6d12d431b5..03749a8668 100644 --- a/mindspore/ccsrc/minddata/dataset/api/datasets.cc +++ b/mindspore/ccsrc/minddata/dataset/api/datasets.cc @@ -483,7 +483,6 @@ FilterDataset::FilterDataset(std::shared_ptr input, std::function input, std::vector> operations, const std::vector &input_columns, const std::vector &output_columns, const std::vector &project_columns, const std::shared_ptr &cache, diff --git a/mindspore/ccsrc/minddata/dataset/api/execute.cc b/mindspore/ccsrc/minddata/dataset/api/execute.cc index 7f27a2a07e..a4dc135057 100644 --- a/mindspore/ccsrc/minddata/dataset/api/execute.cc +++ b/mindspore/ccsrc/minddata/dataset/api/execute.cc @@ -21,6 +21,7 @@ #include "minddata/dataset/core/tensor_row.h" #include "minddata/dataset/include/tensor.h" #include "minddata/dataset/include/type_id.h" +#include "minddata/dataset/kernels/ir/tensor_operation.h" #include "minddata/dataset/kernels/tensor_op.h" #ifndef ENABLE_ANDROID #include "utils/log_adapter.h" diff --git a/mindspore/ccsrc/minddata/dataset/api/transforms.cc b/mindspore/ccsrc/minddata/dataset/api/transforms.cc index 517e7b5af2..269d209ecc 100644 --- a/mindspore/ccsrc/minddata/dataset/api/transforms.cc +++ b/mindspore/ccsrc/minddata/dataset/api/transforms.cc @@ -16,22 +16,40 @@ #include "minddata/dataset/include/transforms.h" +#include + +#include "minddata/dataset/kernels/ir/data/transforms_ir.h" + namespace mindspore { namespace dataset { // Transform operations for data. namespace transforms { -// FUNCTIONS TO CREATE DATA TRANSFORM OPERATIONS +// API CLASS FOR DATA TRANSFORM OPERATIONS // (In alphabetical order) -// Function to create ComposeOperation. -std::shared_ptr Compose(const std::vector> &transforms) { - auto op = std::make_shared(transforms); - // Input validation - return op->ValidateParams() ? op : nullptr; +// Constructor to Compose. +Compose::Compose(const std::vector &transforms) { + (void)std::transform( + transforms.begin(), transforms.end(), std::back_inserter(transforms_), + [](TensorTransform *op) -> std::shared_ptr { return op != nullptr ? op->Parse() : nullptr; }); +} + +Compose::Compose(const std::vector> &transforms) { + (void)std::transform(transforms.begin(), transforms.end(), std::back_inserter(transforms_), + [](std::shared_ptr op) -> std::shared_ptr { + return op != nullptr ? op->Parse() : nullptr; + }); +} + +Compose::Compose(const std::vector> &transforms) { + (void)std::transform(transforms.begin(), transforms.end(), std::back_inserter(transforms_), + [](TensorTransform &op) -> std::shared_ptr { return op.Parse(); }); } +std::shared_ptr Compose::Parse() { return std::make_shared(transforms_); } + // Constructor to Duplicate Duplicate::Duplicate() {} @@ -42,31 +60,66 @@ OneHot::OneHot(int32_t num_classes) : num_classes_(num_classes) {} std::shared_ptr OneHot::Parse() { return std::make_shared(num_classes_); } -// Function to create RandomApplyOperation. -std::shared_ptr RandomApply(const std::vector> &transforms, - double prob) { - auto op = std::make_shared(transforms, prob); - // Input validation - return op->ValidateParams() ? op : nullptr; +// Constructor to RandomApply. +RandomApply::RandomApply(const std::vector &transforms, double prob) : prob_(prob) { + (void)std::transform( + transforms.begin(), transforms.end(), std::back_inserter(transforms_), + [](TensorTransform *op) -> std::shared_ptr { return op != nullptr ? op->Parse() : nullptr; }); +} + +RandomApply::RandomApply(const std::vector> &transforms, double prob) : prob_(prob) { + (void)std::transform(transforms.begin(), transforms.end(), std::back_inserter(transforms_), + [](std::shared_ptr op) -> std::shared_ptr { + return op != nullptr ? op->Parse() : nullptr; + }); +} + +RandomApply::RandomApply(const std::vector> &transforms, double prob) + : prob_(prob) { + (void)std::transform(transforms.begin(), transforms.end(), std::back_inserter(transforms_), + [](TensorTransform &op) -> std::shared_ptr { return op.Parse(); }); } -// Function to create RandomChoiceOperation. -std::shared_ptr RandomChoice(const std::vector> &transforms) { - auto op = std::make_shared(transforms); - // Input validation - return op->ValidateParams() ? op : nullptr; +std::shared_ptr RandomApply::Parse() { + return std::make_shared(transforms_, prob_); } +// Constructor to RandomChoice. +RandomChoice::RandomChoice(const std::vector &transforms) { + (void)std::transform( + transforms.begin(), transforms.end(), std::back_inserter(transforms_), + [](TensorTransform *op) -> std::shared_ptr { return op != nullptr ? op->Parse() : nullptr; }); +} + +RandomChoice::RandomChoice(const std::vector> &transforms) { + (void)std::transform(transforms.begin(), transforms.end(), std::back_inserter(transforms_), + [](std::shared_ptr op) -> std::shared_ptr { + return op != nullptr ? op->Parse() : nullptr; + }); +} + +RandomChoice::RandomChoice(const std::vector> &transforms) { + (void)std::transform(transforms.begin(), transforms.end(), std::back_inserter(transforms_), + [](TensorTransform &op) -> std::shared_ptr { return op.Parse(); }); +} + +std::shared_ptr RandomChoice::Parse() { return std::make_shared(transforms_); } + // Constructor to TypeCast TypeCast::TypeCast(std::string data_type) : data_type_(data_type) {} std::shared_ptr TypeCast::Parse() { return std::make_shared(data_type_); } -#ifndef ENABLE_ANDROID // Constructor to Unique Unique::Unique() {} +#ifndef ENABLE_ANDROID std::shared_ptr Unique::Parse() { return std::make_shared(); } +#else +std::shared_ptr Unique::Parse() { + MS_LOG(ERROR) << "Unique op is not supported for Android."; + return nullptr; +} #endif } // namespace transforms } // namespace dataset diff --git a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/map_node.cc b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/map_node.cc index 7dc0a75df6..1f5ee31e91 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/map_node.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/map_node.cc @@ -24,7 +24,7 @@ #include "minddata/dataset/engine/datasetops/map_op/map_op.h" #include "minddata/dataset/engine/opt/pass.h" -#include "minddata/dataset/include/transforms.h" +#include "minddata/dataset/kernels/ir/tensor_operation.h" #include "minddata/dataset/util/status.h" namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/engine/opt/pre/cache_validation_pass.cc b/mindspore/ccsrc/minddata/dataset/engine/opt/pre/cache_validation_pass.cc index c88d890c49..bd562ac674 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/opt/pre/cache_validation_pass.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/opt/pre/cache_validation_pass.cc @@ -26,6 +26,7 @@ #include "minddata/dataset/engine/ir/datasetops/source/tf_record_node.h" #include "minddata/dataset/engine/ir/datasetops/take_node.h" #include "minddata/dataset/engine/ir/datasetops/zip_node.h" +#include "minddata/dataset/kernels/ir/tensor_operation.h" namespace mindspore { namespace dataset { diff --git a/mindspore/ccsrc/minddata/dataset/include/datasets.h b/mindspore/ccsrc/minddata/dataset/include/datasets.h index a5254a42fd..ef50e9859b 100644 --- a/mindspore/ccsrc/minddata/dataset/include/datasets.h +++ b/mindspore/ccsrc/minddata/dataset/include/datasets.h @@ -295,14 +295,18 @@ class Dataset : public std::enable_shared_from_this { /// \param[in] project_columns A list of column names to project /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). /// \return Shared pointer to the current MapDataset - std::shared_ptr Map(std::vector> operations, + std::shared_ptr Map(std::vector operations, const std::vector &input_columns = {}, const std::vector &output_columns = {}, const std::vector &project_columns = {}, const std::shared_ptr &cache = nullptr, std::vector> callbacks = {}) { - return std::make_shared(shared_from_this(), operations, input_columns, output_columns, project_columns, - cache, callbacks); + std::vector> transform_ops; + (void)std::transform( + operations.begin(), operations.end(), std::back_inserter(transform_ops), + [](TensorTransform *op) -> std::shared_ptr { return op != nullptr ? op->Parse() : nullptr; }); + return std::make_shared(shared_from_this(), transform_ops, input_columns, output_columns, + project_columns, cache, callbacks); } std::shared_ptr Map(std::vector> operations, @@ -312,9 +316,10 @@ class Dataset : public std::enable_shared_from_this { const std::shared_ptr &cache = nullptr, std::vector> callbacks = {}) { std::vector> transform_ops; - (void)std::transform( - operations.begin(), operations.end(), std::back_inserter(transform_ops), - [](std::shared_ptr op) -> std::shared_ptr { return op->Parse(); }); + (void)std::transform(operations.begin(), operations.end(), std::back_inserter(transform_ops), + [](std::shared_ptr op) -> std::shared_ptr { + return op != nullptr ? op->Parse() : nullptr; + }); return std::make_shared(shared_from_this(), transform_ops, input_columns, output_columns, project_columns, cache, callbacks); } diff --git a/mindspore/ccsrc/minddata/dataset/include/text.h b/mindspore/ccsrc/minddata/dataset/include/text.h index 5524fcf91d..a0f6c44c77 100644 --- a/mindspore/ccsrc/minddata/dataset/include/text.h +++ b/mindspore/ccsrc/minddata/dataset/include/text.h @@ -60,7 +60,7 @@ class BasicTokenizer : public TensorTransform { ~BasicTokenizer() = default; /// \brief Function to convert TensorTransform object into a TensorOperation object. - /// \return return code + /// \return Shared pointer to TensorOperation object. std::shared_ptr Parse() override; private: @@ -100,7 +100,7 @@ class BertTokenizer : public TensorTransform { ~BertTokenizer() = default; /// \brief Function to convert TensorTransform object into a TensorOperation object. - /// \return return code + /// \return Shared pointer to TensorOperation object. std::shared_ptr Parse() override; private: @@ -126,7 +126,7 @@ class CaseFold : public TensorTransform { ~CaseFold() = default; /// \brief Function to convert TensorTransform object into a TensorOperation object. - /// \return return code + //// \return Shared pointer to TensorOperation object. std::shared_ptr Parse() override; }; #endif @@ -152,7 +152,7 @@ class JiebaTokenizer : public TensorTransform { ~JiebaTokenizer() = default; /// \brief Function to convert TensorTransform object into a TensorOperation object. - /// \return return code + /// \return Shared pointer to TensorOperation object. std::shared_ptr Parse() override; Status AddWord(const std::string &word, int64_t freq = 0); @@ -181,7 +181,7 @@ class Lookup : public TensorTransform { ~Lookup() = default; /// \brief Function to convert TensorTransform object into a TensorOperation object. - /// \return return code + /// \return Shared pointer to TensorOperation object. std::shared_ptr Parse() override; private: @@ -209,7 +209,7 @@ class Ngram : public TensorTransform { ~Ngram() = default; /// \brief Function to convert TensorTransform object into a TensorOperation object. - /// \return return code + /// \return Shared pointer to TensorOperation object. std::shared_ptr Parse() override; private: @@ -239,7 +239,7 @@ class NormalizeUTF8 : public TensorTransform { ~NormalizeUTF8() = default; /// \brief Function to convert TensorTransform object into a TensorOperation object. - /// \return return code + /// \return Shared pointer to TensorOperation object. std::shared_ptr Parse() override; private: @@ -260,7 +260,7 @@ class RegexReplace : public TensorTransform { ~RegexReplace() = default; /// \brief Function to convert TensorTransform object into a TensorOperation object. - /// \return return code + /// \return Shared pointer to TensorOperation object. std::shared_ptr Parse() override; private: @@ -284,7 +284,7 @@ class RegexTokenizer : public TensorTransform { ~RegexTokenizer() = default; /// \brief Function to convert TensorTransform object into a TensorOperation object. - /// \return return code + /// \return Shared pointer to TensorOperation object. std::shared_ptr Parse() override; private: @@ -312,7 +312,7 @@ class SentencePieceTokenizer : public TensorTransform { ~SentencePieceTokenizer() = default; /// \brief Function to convert TensorTransform object into a TensorOperation object. - /// \return return code + /// \return Shared pointer to TensorOperation object. std::shared_ptr Parse() override; private: @@ -336,7 +336,7 @@ class SlidingWindow : public TensorTransform { ~SlidingWindow() = default; /// \brief Function to convert TensorTransform object into a TensorOperation object. - /// \return return code + /// \return Shared pointer to TensorOperation object. std::shared_ptr Parse() override; private: @@ -359,7 +359,7 @@ class ToNumber : public TensorTransform { ~ToNumber() = default; /// \brief Function to convert TensorTransform object into a TensorOperation object. - /// \return return code + /// \return Shared pointer to TensorOperation object. std::shared_ptr Parse() override; private: @@ -377,7 +377,7 @@ class TruncateSequencePair : public TensorTransform { ~TruncateSequencePair() = default; /// \brief Function to convert TensorTransform object into a TensorOperation object. - /// \return return code + /// \return Shared pointer to TensorOperation object. std::shared_ptr Parse() override; private: @@ -395,7 +395,7 @@ class UnicodeCharTokenizer : public TensorTransform { ~UnicodeCharTokenizer() = default; /// \brief Function to convert TensorTransform object into a TensorOperation object. - /// \return return code + /// \return Shared pointer to TensorOperation object. std::shared_ptr Parse() override; private: @@ -415,7 +415,7 @@ class UnicodeScriptTokenizer : public TensorTransform { ~UnicodeScriptTokenizer() = default; /// \brief Function to convert TensorTransform object into a TensorOperation object. - /// \return return code + /// \return Shared pointer to TensorOperation object. std::shared_ptr Parse() override; private: @@ -434,7 +434,7 @@ class WhitespaceTokenizer : public TensorTransform { ~WhitespaceTokenizer() = default; /// \brief Function to convert TensorTransform object into a TensorOperation object. - /// \return return code + /// \return Shared pointer to TensorOperation object. std::shared_ptr Parse() override; private: diff --git a/mindspore/ccsrc/minddata/dataset/include/transforms.h b/mindspore/ccsrc/minddata/dataset/include/transforms.h index 0c58c0086c..fce85e30b5 100644 --- a/mindspore/ccsrc/minddata/dataset/include/transforms.h +++ b/mindspore/ccsrc/minddata/dataset/include/transforms.h @@ -25,11 +25,11 @@ #include "include/api/status.h" #include "minddata/dataset/include/constants.h" -// FIXME - This internal IR header will be removed when external API classes are provided -#include "minddata/dataset/kernels/ir/data/transforms_ir.h" - namespace mindspore { namespace dataset { + +class TensorOperation; + // Abstract class to represent a tensor transform operation in the data pipeline. /// \class TensorTransform transforms.h /// \brief A base class to represent a tensor transform operation in the data pipeline. @@ -54,16 +54,26 @@ class TensorTransform : public std::enable_shared_from_this { // Transform operations for performing data transformation. namespace transforms { -// Transform Op classes (in alphabetical order) -class ComposeOperation; -class RandomApplyOperation; -class RandomChoiceOperation; - -/// \brief Function to create a Compose TensorOperation. +/// \brief Compose Op. /// \notes Compose a list of transforms into a single transform. -/// \param[in] transforms A vector of transformations to be applied. -/// \return Shared pointer to the current TensorOperation. -std::shared_ptr Compose(const std::vector> &transforms); +class Compose : public TensorTransform { + public: + /// \brief Constructor. + /// \param[in] transforms A vector of transformations to be applied. + explicit Compose(const std::vector &transforms); + explicit Compose(const std::vector> &transforms); + explicit Compose(const std::vector> &transforms); + + /// \brief Destructor + ~Compose() = default; + + /// \brief Function to convert TensorTransform object into a TensorOperation object. + /// \return Shared pointer to TensorOperation object. + std::shared_ptr Parse() override; + + private: + std::vector> transforms_; +}; /// \brief Duplicate Op. /// \notes Duplicate the input tensor to a new output tensor. @@ -77,7 +87,7 @@ class Duplicate : public TensorTransform { ~Duplicate() = default; /// \brief Function to convert TensorTransform object into a TensorOperation object. - /// \return return code + /// \return Shared pointer to TensorOperation object. std::shared_ptr Parse() override; }; @@ -93,26 +103,56 @@ class OneHot : public TensorTransform { ~OneHot() = default; /// \brief Function to convert TensorTransform object into a TensorOperation object. - /// \return return code + /// \return Shared pointer to TensorOperation object. std::shared_ptr Parse() override; private: float num_classes_; }; -/// \brief Function to create a RandomApply TensorOperation. +/// \brief RandomApply Op. /// \notes Randomly perform a series of transforms with a given probability. -/// \param[in] transforms A vector of transformations to be applied. -/// \param[in] prob The probability to apply the transformation list (default=0.5) -/// \return Shared pointer to the current TensorOperation. -std::shared_ptr RandomApply(const std::vector> &transforms, - double prob = 0.5); +class RandomApply : public TensorTransform { + public: + /// \brief Constructor. + /// \param[in] transforms A vector of transformations to be applied. + /// \param[in] prob The probability to apply the transformation list (default=0.5) + explicit RandomApply(const std::vector &transforms, double prob = 0.5); + explicit RandomApply(const std::vector> &transforms, double prob = 0.5); + explicit RandomApply(const std::vector> &transforms, double prob = 0.5); + + /// \brief Destructor + ~RandomApply() = default; -/// \brief Function to create a RandomChoice TensorOperation. + /// \brief Function to convert TensorTransform object into a TensorOperation object. + /// \return Shared pointer to TensorOperation object. + std::shared_ptr Parse() override; + + private: + std::vector> transforms_; + double prob_; +}; + +/// \brief RandomChoice Op. /// \notes Randomly selects one transform from a list of transforms to perform operation. -/// \param[in] transforms A vector of transformations to be chosen from to apply. -/// \return Shared pointer to the current TensorOperation. -std::shared_ptr RandomChoice(const std::vector> &transforms); +class RandomChoice : public TensorTransform { + public: + /// \brief Constructor. + /// \param[in] transforms A vector of transformations to be chosen from to apply. + explicit RandomChoice(const std::vector &transforms); + explicit RandomChoice(const std::vector> &transforms); + explicit RandomChoice(const std::vector> &transforms); + + /// \brief Destructor + ~RandomChoice() = default; + + /// \brief Function to convert TensorTransform object into a TensorOperation object. + /// \return Shared pointer to TensorOperation object. + std::shared_ptr Parse() override; + + private: + std::vector> transforms_; +}; /// \brief TypeCast Op. /// \notes Tensor operation to cast to a given MindSpore data type. @@ -126,14 +166,13 @@ class TypeCast : public TensorTransform { ~TypeCast() = default; /// \brief Function to convert TensorTransform object into a TensorOperation object. - /// \return return code + /// \return Shared pointer to TensorOperation object. std::shared_ptr Parse() override; private: std::string data_type_; }; -#ifndef ENABLE_ANDROID /// \brief Unique Op. /// \notes Return an output tensor containing all the unique elements of the input tensor in /// the same order that they occur in the input tensor. @@ -146,10 +185,9 @@ class Unique : public TensorTransform { ~Unique() = default; /// \brief Function to convert TensorTransform object into a TensorOperation object. - /// \return return code + /// \return Shared pointer to TensorOperation object. std::shared_ptr Parse() override; }; -#endif } // namespace transforms } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/kernels/ir/validators.cc b/mindspore/ccsrc/minddata/dataset/kernels/ir/validators.cc index 8967927b9f..4bfdaccd69 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/ir/validators.cc +++ b/mindspore/ccsrc/minddata/dataset/kernels/ir/validators.cc @@ -184,6 +184,8 @@ Status ValidateVectorTransforms(const std::string &op_name, op_name + ": transform ops must not be null, got transform[" + std::to_string(i) + "] == nullptr."; MS_LOG(ERROR) << err_msg; RETURN_STATUS_SYNTAX_ERROR(err_msg); + } else { + RETURN_IF_NOT_OK(transforms[i]->ValidateParams()); } } diff --git a/mindspore/ccsrc/minddata/dataset/kernels/ir/validators.h b/mindspore/ccsrc/minddata/dataset/kernels/ir/validators.h index 555a4369c7..56e058829d 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/ir/validators.h +++ b/mindspore/ccsrc/minddata/dataset/kernels/ir/validators.h @@ -22,13 +22,12 @@ #include #include "minddata/dataset/core/tensor.h" +#include "minddata/dataset/kernels/ir/tensor_operation.h" #include "minddata/dataset/util/status.h" namespace mindspore { namespace dataset { -class TensorOperation; - // Helper function to validate probability Status ValidateProbability(const std::string &op_name, const float probability); diff --git a/tests/ut/cpp/dataset/c_api_dataset_iterator_test.cc b/tests/ut/cpp/dataset/c_api_dataset_iterator_test.cc index 2a7b53a467..cd8bd91d7c 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_iterator_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_iterator_test.cc @@ -166,7 +166,7 @@ TEST_F(MindDataTestPipeline, TestIteratorTwoColumns) { // TensorShape({211653}), TensorShape({1, 4})}; uint64_t i = 0; - uint64_t j = 0; + // uint64_t j = 0; while (row.size() != 0) { // MS_LOG(INFO) << "row[0]:" << row[0]->shape() << ", row[1]:" << row[1]->shape(); // EXPECT_EQ(2, row.size()); diff --git a/tests/ut/cpp/dataset/c_api_dataset_manifest_test.cc b/tests/ut/cpp/dataset/c_api_dataset_manifest_test.cc index 4e98402fac..ee7ec08c6a 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_manifest_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_manifest_test.cc @@ -238,7 +238,7 @@ TEST_F(MindDataTestPipeline, TestManifestClassIndex) { iter->GetNextRow(&row); uint64_t i = 0; - int32_t label_idx = 0; + // int32_t label_idx = 0; while (row.size() != 0) { i++; // auto image = row["image"]; diff --git a/tests/ut/cpp/dataset/c_api_transforms_test.cc b/tests/ut/cpp/dataset/c_api_transforms_test.cc index dce9cce978..c453ac6d40 100644 --- a/tests/ut/cpp/dataset/c_api_transforms_test.cc +++ b/tests/ut/cpp/dataset/c_api_transforms_test.cc @@ -35,10 +35,11 @@ TEST_F(MindDataTestPipeline, TestComposeSuccess) { std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::shared_ptr ds = ImageFolder(folder_path, false, RandomSampler(false, 3)); EXPECT_NE(ds, nullptr); - /* FIXME - Disable until proper external API for Compose is provided + // Create objects for the tensor ops - std::shared_ptr compose = transforms::Compose({vision::Decode(), vision::Resize({777, 777})}); - EXPECT_NE(compose, nullptr); + auto decode_op(new vision::Decode()); + auto resize_op(new vision::Resize({777, 777})); + transforms::Compose compose({decode_op, resize_op}); // Create a Map operation on ds ds = ds->Map({compose}, {"image"}); @@ -69,25 +70,71 @@ TEST_F(MindDataTestPipeline, TestComposeSuccess) { // Manually terminate the pipeline iter->Stop(); - */ } -TEST_F(MindDataTestPipeline, TestComposeFail) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestComposeFail with invalid transform."; - /* FIXME - Disable until proper external API for Compose is provided +TEST_F(MindDataTestPipeline, TestComposeFail1) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestComposeFail1 with invalid transform."; + + // Create a Cifar10 Dataset + std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; + std::shared_ptr ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); + EXPECT_NE(ds, nullptr); + // Resize: Non-positive size value: -1 at element: 0 // Compose: transform ops must not be null - std::shared_ptr compose1 = transforms::Compose({vision::Decode(), vision::Resize({-1})}); - EXPECT_EQ(compose1, nullptr); + auto decode_op = vision::Decode(); + auto resize_op = vision::Resize({-1}); + auto compose = transforms::Compose({decode_op, resize_op}); + + // Create a Map operation on ds + ds = ds->Map({compose}, {"image"}); + EXPECT_NE(ds, nullptr); + + std::shared_ptr iter = ds->CreateIterator(); + // Expect failure: invalid Compose parameter(invalid transform op) + EXPECT_EQ(iter, nullptr); +} + +TEST_F(MindDataTestPipeline, TestComposeFail2) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestComposeFail2 with invalid transform."; + + // Create a Cifar10 Dataset + std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; + std::shared_ptr ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); + EXPECT_NE(ds, nullptr); // Compose: transform ops must not be null - std::shared_ptr compose2 = transforms::Compose({vision::Decode(), nullptr}); - EXPECT_EQ(compose2, nullptr); + std::shared_ptr decode_op = std::make_shared(); + std::shared_ptr compose(new transforms::Compose({decode_op, nullptr})); + + // Create a Map operation on ds + ds = ds->Map({compose}, {"image"}); + EXPECT_NE(ds, nullptr); + + std::shared_ptr iter = ds->CreateIterator(); + // Expect failure: invalid Compose parameter (transform ops must not be null) + EXPECT_EQ(iter, nullptr); +} + +TEST_F(MindDataTestPipeline, TestComposeFail3) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestComposeFail3 with invalid transform."; + + // Create a Cifar10 Dataset + std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; + std::shared_ptr ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); + EXPECT_NE(ds, nullptr); // Compose: transform list must not be empty - std::shared_ptr compose3 = transforms::Compose({}); - EXPECT_EQ(compose3, nullptr); - */ + std::vector> list = {}; + auto compose = transforms::Compose(list); + + // Create a Map operation on ds + ds = ds->Map({compose}, {"image"}); + EXPECT_NE(ds, nullptr); + + std::shared_ptr iter = ds->CreateIterator(); + // Expect failure: invalid Compose parameter (transform list must not be empty) + EXPECT_EQ(iter, nullptr); } TEST_F(MindDataTestPipeline, TestDuplicateSuccess) { @@ -296,10 +343,10 @@ TEST_F(MindDataTestPipeline, TestRandomApplySuccess) { std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 5)); EXPECT_NE(ds, nullptr); - /* FIXME - Disable until proper external API for RandomApply is provided + // Create objects for the tensor ops - std::shared_ptr random_apply = transforms::RandomApply({vision::Resize({777, 777})}, 0.8); - EXPECT_NE(random_apply, nullptr); + auto resize_op = vision::Resize({777, 777}); + auto random_apply = transforms::RandomApply({resize_op}, 0.8); // Create a Map operation on ds ds = ds->Map({random_apply}, {"image"}); @@ -328,29 +375,92 @@ TEST_F(MindDataTestPipeline, TestRandomApplySuccess) { // Manually terminate the pipeline iter->Stop(); - */ } -TEST_F(MindDataTestPipeline, TestRandomApplyFail) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomApplyFail with invalid transform."; - /* FIXME - Disable until proper external API for RandomApply is provided +TEST_F(MindDataTestPipeline, TestRandomApplyFail1) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomApplyFail1 with invalid transform."; + + // Create a Cifar10 Dataset + std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; + std::shared_ptr ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); + EXPECT_NE(ds, nullptr); + // Resize: Non-positive size value: -1 at element: 0 // RandomApply: transform ops must not be null - std::shared_ptr random_apply1 = transforms::RandomApply({vision::Decode(), vision::Resize({-1})}); - EXPECT_EQ(random_apply1, nullptr); + auto decode_op = vision::Decode(); + auto resize_op = vision::Resize({-1}); + auto random_apply = transforms::RandomApply({decode_op, resize_op}); + + // Create a Map operation on ds + ds = ds->Map({random_apply}, {"image"}); + EXPECT_NE(ds, nullptr); + + std::shared_ptr iter = ds->CreateIterator(); + // Expect failure: invalid RandomApply parameter (transform ops must not be null) + EXPECT_EQ(iter, nullptr); +} + +TEST_F(MindDataTestPipeline, TestRandomApplyFail2) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomApplyFail2 with invalid transform."; + + // Create a Cifar10 Dataset + std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; + std::shared_ptr ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); + EXPECT_NE(ds, nullptr); // RandomApply: transform ops must not be null - std::shared_ptr random_apply2 = transforms::RandomApply({vision::Decode(), nullptr}); - EXPECT_EQ(random_apply2, nullptr); + std::shared_ptr decode_op = std::make_shared(); + std::shared_ptr random_apply(new transforms::RandomApply({decode_op, nullptr})); - // RandomApply: transform list must not be empty - std::shared_ptr random_apply3 = transforms::RandomApply({}); - EXPECT_EQ(random_apply3, nullptr); + // Create a Map operation on ds + ds = ds->Map({random_apply}, {"image"}); + EXPECT_NE(ds, nullptr); + + std::shared_ptr iter = ds->CreateIterator(); + // Expect failure: invalid RandomApply parameter (transform ops must not be null) + EXPECT_EQ(iter, nullptr); +} + +TEST_F(MindDataTestPipeline, TestRandomApplyFail3) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomApplyFail3 with invalid transform."; + + // Create a Cifar10 Dataset + std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; + std::shared_ptr ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); + EXPECT_NE(ds, nullptr); // RandomApply: Probability has to be between 0 and 1 - std::shared_ptr random_apply4 = transforms::RandomApply({vision::Resize({100})}, -1); - EXPECT_EQ(random_apply4, nullptr); - */ + auto resize_op = vision::Resize({100}); + auto random_apply = transforms::RandomApply({resize_op}, -1); + + // Create a Map operation on ds + ds = ds->Map({random_apply}, {"image"}); + EXPECT_NE(ds, nullptr); + + std::shared_ptr iter = ds->CreateIterator(); + // Expect failure: invalid RandomApply parameter (Probability has to be between 0 and 1) + EXPECT_EQ(iter, nullptr); +} + +TEST_F(MindDataTestPipeline, TestRandomApplyFail4) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomApplyFail4 with invalid transform."; + + // Create a Cifar10 Dataset + std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; + std::shared_ptr ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); + EXPECT_NE(ds, nullptr); + + // RandomApply: transform list must not be empty + std::vector> list = {}; + auto random_apply = transforms::RandomApply(list); + + // Create a Map operation on ds + ds = ds->Map({random_apply}, {"image"}); + EXPECT_NE(ds, nullptr); + + std::shared_ptr iter = ds->CreateIterator(); + // Expect failure: invalid RandomApply parameter (transform list must not be empty) + EXPECT_EQ(iter, nullptr); } TEST_F(MindDataTestPipeline, TestRandomChoiceSuccess) { @@ -360,11 +470,11 @@ TEST_F(MindDataTestPipeline, TestRandomChoiceSuccess) { std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 3)); EXPECT_NE(ds, nullptr); - /* FIXME - Disable until proper external API for RandomChoice is provided + // Create objects for the tensor ops - std::shared_ptr random_choice = - transforms::RandomChoice({vision::Resize({777, 777}), vision::Resize({888, 888})}); - EXPECT_NE(random_choice, nullptr); + std::shared_ptr resize_op1(new vision::Resize({777, 777})); + std::shared_ptr resize_op2(new vision::Resize({888, 888})); + auto random_choice = transforms::RandomChoice({resize_op1, resize_op2}); // Create a Map operation on ds ds = ds->Map({random_choice}, {"image"}); @@ -393,25 +503,71 @@ TEST_F(MindDataTestPipeline, TestRandomChoiceSuccess) { // Manually terminate the pipeline iter->Stop(); - */ } -TEST_F(MindDataTestPipeline, TestRandomChoiceFail) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomChoiceFail with invalid transform."; - /* FIXME - Disable until proper external API for RandomChoice is provided +TEST_F(MindDataTestPipeline, TestRandomChoiceFail1) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomChoiceFail1 with invalid transform."; + + // Create a Cifar10 Dataset + std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; + std::shared_ptr ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); + EXPECT_NE(ds, nullptr); + // Resize: Non-positive size value: -1 at element: 0 // RandomChoice: transform ops must not be null - std::shared_ptr random_choice1 = transforms::RandomChoice({vision::Decode(), vision::Resize({-1})}); - EXPECT_EQ(random_choice1, nullptr); + auto decode_op = vision::Decode(); + auto resize_op = vision::Resize({-1}); + auto random_choice = transforms::RandomChoice({decode_op, resize_op}); + + // Create a Map operation on ds + ds = ds->Map({random_choice}, {"image"}); + EXPECT_NE(ds, nullptr); + + std::shared_ptr iter = ds->CreateIterator(); + // Expect failure: invalid RandomApply parameter (transform ops must not be null) + EXPECT_EQ(iter, nullptr); +} + +TEST_F(MindDataTestPipeline, TestRandomChoiceFail2) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomChoiceFail2 with invalid transform."; + + // Create a Cifar10 Dataset + std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; + std::shared_ptr ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); + EXPECT_NE(ds, nullptr); // RandomChoice: transform ops must not be null - std::shared_ptr random_choice2 = transforms::RandomChoice({vision::Decode(), nullptr}); - EXPECT_EQ(random_choice2, nullptr); + std::shared_ptr decode_op = std::make_shared(); + std::shared_ptr random_choice(new transforms::RandomApply({decode_op, nullptr})); + + // Create a Map operation on ds + ds = ds->Map({random_choice}, {"image"}); + EXPECT_NE(ds, nullptr); + + std::shared_ptr iter = ds->CreateIterator(); + // Expect failure: invalid RandomApply parameter (transform ops must not be null) + EXPECT_EQ(iter, nullptr); +} + +TEST_F(MindDataTestPipeline, TestRandomChoiceFail3) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomChoiceFail3 with invalid transform."; + + // Create a Cifar10 Dataset + std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; + std::shared_ptr ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); + EXPECT_NE(ds, nullptr); // RandomChoice: transform list must not be empty - std::shared_ptr random_choice3 = transforms::RandomChoice({}); - EXPECT_EQ(random_choice3, nullptr); - */ + std::vector> list = {}; + auto random_choice = transforms::RandomChoice(list); + + // Create a Map operation on ds + ds = ds->Map({random_choice}, {"image"}); + EXPECT_NE(ds, nullptr); + + std::shared_ptr iter = ds->CreateIterator(); + // Expect failure: invalid RandomApply parameter (transform list must not be empty) + EXPECT_EQ(iter, nullptr); } TEST_F(MindDataTestPipeline, TestTypeCastSuccess) { diff --git a/tests/ut/cpp/dataset/optimization_pass_test.cc b/tests/ut/cpp/dataset/optimization_pass_test.cc index eb4b82ad52..68932793f8 100644 --- a/tests/ut/cpp/dataset/optimization_pass_test.cc +++ b/tests/ut/cpp/dataset/optimization_pass_test.cc @@ -27,6 +27,8 @@ #include "minddata/dataset/include/transforms.h" #include "minddata/dataset/include/vision.h" #include "minddata/dataset/include/vision_lite.h" +#include "minddata/dataset/kernels/ir/data/transforms_ir.h" +#include "minddata/dataset/kernels/ir/vision/vision_ir.h" using namespace mindspore::dataset; using mindspore::LogStream; @@ -42,8 +44,7 @@ TEST_F(MindDataTestOptimizationPass, MindDataTestAutoWorkerPass) { std::shared_ptr map_leaf = ImageFolder("dir")->SetNumWorkers(0); std::shared_ptr nonmap_leaf = RandomData(44, schema)->SetNumWorkers(0); std::shared_ptr batch = Zip({map_leaf, nonmap_leaf})->Batch(1)->SetNumWorkers(0); - /* FIXME - Will uncomment out when full external API support is provided - std::shared_ptr map = batch->Map({})->SetNumWorkers(0); + std::shared_ptr map = batch->Map({std::shared_ptr()})->SetNumWorkers(0); // {ImageFolder, RandomData} -> zip -> batch EXPECT_EQ(map_leaf->IRNode()->num_workers(), 0); EXPECT_EQ(nonmap_leaf->IRNode()->num_workers(), 0); @@ -65,15 +66,14 @@ TEST_F(MindDataTestOptimizationPass, MindDataTestAutoWorkerPass) { MS_LOG(DEBUG) << nonmap_leaf->IRNode()->Name() << ": num_worker=" << nonmap_leaf->IRNode()->num_workers(); MS_LOG(DEBUG) << batch->IRNode()->Name() << ": num_worker=" << batch->IRNode()->num_workers(); MS_LOG(DEBUG) << map->IRNode()->Name() << ": num_worker=" << map->IRNode()->num_workers(); - */ } TEST_F(MindDataTestOptimizationPass, MindDataTestTensorFusionPass) { MS_LOG(INFO) << "Doing MindDataTestOptimizationPass-MindDataTestTensorFusionPass."; std::string folder_path = datasets_root_path_ + "/testPK/data/"; - /* FIXME - Will uncomment out when full external API support is provided - std::shared_ptr root = - ImageFolder(folder_path, false)->Map({vision::Decode(), vision::RandomResizedCrop({100})}, {"image"}); + auto decode_op = vision::Decode(); + auto random_resized_crop_op = vision::RandomResizedCrop({100}); + std::shared_ptr root = ImageFolder(folder_path, false)->Map({decode_op, random_resized_crop_op}, {"image"}); TensorOpFusionPass fusion_pass; bool modified = false; @@ -85,27 +85,26 @@ TEST_F(MindDataTestOptimizationPass, MindDataTestTensorFusionPass) { auto fused_ops = map_node->operations(); ASSERT_EQ(fused_ops.size(), 1); ASSERT_EQ(fused_ops[0]->Name(), vision::kRandomCropDecodeResizeOperation); - */ } TEST_F(MindDataTestOptimizationPass, MindDataTestTensorFusionPassPreBuiltTensorOperation) { MS_LOG(INFO) << "Doing MindDataTestOptimizationPass-MindDataTestTensorFusionPassPreBuiltTensorOperation."; std::string folder_path = datasets_root_path_ + "/testPK/data/"; - /* FIXME - Will uncomment out when full external API support is provided // make prebuilt tensor operation - auto decode = std::make_shared(vision::Decode()->Build()); - auto resize = std::make_shared(vision::RandomResizedCrop({100})->Build()); - std::shared_ptr root = ImageFolder(folder_path, false)->Map({decode, resize}, {"image"}); + auto decode = std::make_shared(vision::Decode().Parse()->Build()); + auto resize = std::make_shared(vision::RandomResizedCrop({100}).Parse()->Build()); + std::vector> op_list = {decode, resize}; + std::vector op_name = {"image"}; + std::shared_ptr root = ImageFolder(folder_path, false)->IRNode(); + std::shared_ptr map_node = std::make_shared(root, op_list, op_name); TensorOpFusionPass fusion_pass; bool modified = false; - std::shared_ptr map_node = std::dynamic_pointer_cast(root->IRNode()); // no deepcopy is performed because this doesn't go through tree_adapter - fusion_pass.Run(root->IRNode(), &modified); + fusion_pass.Run(map_node, &modified); EXPECT_EQ(modified, true); ASSERT_NE(map_node, nullptr); auto fused_ops = map_node->operations(); ASSERT_EQ(fused_ops.size(), 1); ASSERT_EQ(fused_ops[0]->Name(), kRandomCropDecodeResizeOp); - */ }