diff --git a/mindspore/ccsrc/minddata/dataset/include/text.h b/mindspore/ccsrc/minddata/dataset/include/text.h index d492334b6b..2f43ab8cf4 100644 --- a/mindspore/ccsrc/minddata/dataset/include/text.h +++ b/mindspore/ccsrc/minddata/dataset/include/text.h @@ -37,6 +37,14 @@ namespace dataset { // Transform operations for text namespace text { +// Char arrays storing name of corresponding classes (in alphabetical order) +constexpr char kJiebaTokenizerOperation[] = "JiebaTokenizer"; +constexpr char kLookupOperation[] = "Lookup"; +constexpr char kNgramOperation[] = "Ngram"; +constexpr char kSentencepieceTokenizerOperation[] = "SentencepieceTokenizer"; +constexpr char kSlidingWindowOperation[] = "SlidingWindow"; +constexpr char kWhitespaceTokenizerOperation[] = "WhitespaceTokenizer"; + // Text Op classes (in alphabetical order) class JiebaTokenizerOperation; class LookupOperation; @@ -128,6 +136,8 @@ class JiebaTokenizerOperation : public TensorOperation { Status ValidateParams() override; + std::string Name() const override { return kJiebaTokenizerOperation; } + private: std::string hmm_path_; std::string mp_path_; @@ -146,6 +156,8 @@ class LookupOperation : public TensorOperation { Status ValidateParams() override; + std::string Name() const override { return kLookupOperation; } + private: std::shared_ptr vocab_; std::string unknown_token_; @@ -164,6 +176,8 @@ class NgramOperation : public TensorOperation { Status ValidateParams() override; + std::string Name() const override { return kNgramOperation; } + private: std::vector ngrams_; std::pair left_pad_; @@ -183,6 +197,8 @@ class SentencePieceTokenizerOperation : public TensorOperation { Status ValidateParams() override; + std::string Name() const override { return kSentencepieceTokenizerOperation; } + private: std::shared_ptr vocab_; std::string vocab_path_; @@ -200,6 +216,8 @@ class SlidingWindowOperation : public TensorOperation { Status ValidateParams() override; + std::string Name() const override { return kSlidingWindowOperation; } + private: int32_t width_; int32_t axis_; @@ -216,6 +234,8 @@ class WhitespaceTokenizerOperation : public TensorOperation { Status ValidateParams() override; + std::string Name() const override { return kWhitespaceTokenizerOperation; } + private: bool with_offsets_; }; diff --git a/mindspore/ccsrc/minddata/dataset/include/transforms.h b/mindspore/ccsrc/minddata/dataset/include/transforms.h index 3c9f624bb6..208976fc52 100644 --- a/mindspore/ccsrc/minddata/dataset/include/transforms.h +++ b/mindspore/ccsrc/minddata/dataset/include/transforms.h @@ -28,6 +28,17 @@ namespace dataset { class TensorOp; +// Char arrays storing name of corresponding classes (in alphabetical order) +constexpr char kComposeOperation[] = "Compose"; +constexpr char kDuplicateOperation[] = "Duplicate"; +constexpr char kOneHotOperation[] = "OneHot"; +constexpr char kPreBuiltOperation[] = "PreBuilt"; +constexpr char kRandomApplyOperation[] = "RandomApply"; +constexpr char kRandomChoiceOperation[] = "RandomChoice"; +constexpr char kRandomSelectSubpolicyOperation[] = "RandomSelectSubpolicy"; +constexpr char kTypeCastOperation[] = "TypeCast"; +constexpr char kUniqueOperation[] = "Unique"; + // Abstract class to represent a dataset in the data pipeline. class TensorOperation : public std::enable_shared_from_this { public: @@ -46,6 +57,8 @@ class TensorOperation : public std::enable_shared_from_this { virtual Status ValidateParams() = 0; + virtual std::string Name() const = 0; + /// \brief Check whether the operation is deterministic. /// \return true if this op is a random op (returns non-deterministic result e.g. RandomCrop) bool IsRandomOp() const { return random_op_; } @@ -146,6 +159,8 @@ class ComposeOperation : public TensorOperation { Status ValidateParams() override; + std::string Name() const override { return kComposeOperation; } + private: std::vector> transforms_; }; @@ -159,6 +174,8 @@ class DuplicateOperation : public TensorOperation { std::shared_ptr Build() override; Status ValidateParams() override; + + std::string Name() const override { return kDuplicateOperation; } }; class OneHotOperation : public TensorOperation { @@ -171,6 +188,8 @@ class OneHotOperation : public TensorOperation { Status ValidateParams() override; + std::string Name() const override { return kOneHotOperation; } + private: float num_classes_; }; @@ -185,6 +204,8 @@ class PreBuiltOperation : public TensorOperation { Status ValidateParams() override; + std::string Name() const override { return kPreBuiltOperation; } + private: std::shared_ptr op_; }; @@ -199,6 +220,8 @@ class RandomApplyOperation : public TensorOperation { Status ValidateParams() override; + std::string Name() const override { return kRandomApplyOperation; } + private: std::vector> transforms_; double prob_; @@ -214,6 +237,8 @@ class RandomChoiceOperation : public TensorOperation { Status ValidateParams() override; + std::string Name() const override { return kRandomChoiceOperation; } + private: std::vector> transforms_; }; @@ -227,6 +252,8 @@ class TypeCastOperation : public TensorOperation { Status ValidateParams() override; + std::string Name() const override { return kTypeCastOperation; } + private: std::string data_type_; }; @@ -241,6 +268,8 @@ class UniqueOperation : public TensorOperation { std::shared_ptr Build() override; Status ValidateParams() override; + + std::string Name() const override { return kUniqueOperation; } }; #endif } // namespace transforms diff --git a/mindspore/ccsrc/minddata/dataset/include/vision.h b/mindspore/ccsrc/minddata/dataset/include/vision.h index 3c352d9628..a4a52651fa 100644 --- a/mindspore/ccsrc/minddata/dataset/include/vision.h +++ b/mindspore/ccsrc/minddata/dataset/include/vision.h @@ -19,6 +19,7 @@ #include #include +#include #include #include #include "minddata/dataset/core/constants.h" @@ -31,6 +32,48 @@ namespace dataset { // Transform operations for performing computer vision. namespace vision { +// Char arrays storing name of corresponding classes (in alphabetical order) +constexpr char kAutoContrastOperation[] = "AutoContrast"; +constexpr char kBoundingBoxAugmentOperation[] = "BoundingBoxAugment"; +constexpr char kCenterCropOperation[] = "CenterCrop"; +constexpr char kCutMixBatchOperation[] = "CutMixBatch"; +constexpr char kCutOutOperation[] = "CutOut"; +constexpr char kCropOperation[] = "Crop"; +constexpr char kDecodeOperation[] = "Decode"; +constexpr char kEqualizeOperation[] = "Equalize"; +constexpr char kHwcToChwOperation[] = "HwcToChw"; +constexpr char kInvertOperation[] = "Invert"; +constexpr char kMixUpBatchOperation[] = "MixUpBatch"; +constexpr char kNormalizeOperation[] = "Normalize"; +constexpr char kPadOperation[] = "Pad"; +constexpr char kRandomAffineOperation[] = "RandomAffine"; +constexpr char kRandomColorAdjustOperation[] = "RandomColorAdjust"; +constexpr char kRandomColorOperation[] = "RandomColor"; +constexpr char kRandomCropDecodeResizeOperation[] = "RandomCropDecodeResize"; +constexpr char kRandomCropOperation[] = "RandomCrop"; +constexpr char kRandomCropWithBBoxOperation[] = "RandomCropWithBBox"; +constexpr char kRandomHorizontalFlipWithBBoxOperation[] = "RandomHorizontalFlipWithBBox"; +constexpr char kRandomHorizontalFlipOperation[] = "RandomHorizontalFlip"; +constexpr char kRandomPosterizeOperation[] = "RandomPosterize"; +constexpr char kRandomResizedCropOperation[] = "RandomResizedCrop"; +constexpr char kRandomResizedCropWithBBoxOperation[] = "RandomResizedCropWithBBox"; +constexpr char kRandomResizeOperation[] = "RandomResize"; +constexpr char kRandomResizeWithBBoxOperation[] = "RandomResizeWithBBox"; +constexpr char kRandomRotationOperation[] = "RandomRotation"; +constexpr char kRandomSolarizeOperation[] = "RandomSolarize"; +constexpr char kRandomSharpnessOperation[] = "RandomSharpness"; +constexpr char kRandomVerticalFlipOperation[] = "RandomVerticalFlip"; +constexpr char kRandomVerticalFlipWithBBoxOperation[] = "RandomVerticalFlipWithBBox"; +constexpr char kRescaleOperation[] = "Rescale"; +constexpr char kResizeOperation[] = "Resize"; +constexpr char kResizeWithBBoxOperation[] = "ResizeWithBBox"; +constexpr char kRgbaToBgrOperation[] = "RgbaToBgr"; +constexpr char kRgbaToRgbOperation[] = "RgbaToRgb"; +constexpr char kSoftDvppDecodeRandomCropResizeJpegOperation[] = "SoftDvppDecodeRandomCropResizeJpeg"; +constexpr char kSoftDvppDecodeResizeJpegOperation[] = "SoftDvppDecodeResizeJpeg"; +constexpr char kSwapRedBlueOperation[] = "SwapRedBlue"; +constexpr char kUniformAugOperation[] = "UniformAug"; + // Transform Op classes (in alphabetical order) #ifndef ENABLE_ANDROID class AutoContrastOperation; @@ -501,6 +544,8 @@ class AutoContrastOperation : public TensorOperation { Status ValidateParams() override; + std::string Name() const override { return kAutoContrastOperation; } + private: float cutoff_; std::vector ignore_; @@ -516,6 +561,8 @@ class BoundingBoxAugmentOperation : public TensorOperation { Status ValidateParams() override; + std::string Name() const override { return kBoundingBoxAugmentOperation; } + private: std::shared_ptr transform_; float ratio_; @@ -533,6 +580,8 @@ class CenterCropOperation : public TensorOperation { Status ValidateParams() override; + std::string Name() const override { return kCenterCropOperation; } + private: std::vector size_; }; @@ -547,6 +596,8 @@ class CropOperation : public TensorOperation { Status ValidateParams() override; + std::string Name() const override { return kCropOperation; } + private: std::vector coordinates_; std::vector size_; @@ -562,6 +613,8 @@ class CutMixBatchOperation : public TensorOperation { Status ValidateParams() override; + std::string Name() const override { return kCutMixBatchOperation; } + private: float alpha_; float prob_; @@ -578,6 +631,8 @@ class CutOutOperation : public TensorOperation { Status ValidateParams() override; + std::string Name() const override { return kCutOutOperation; } + private: int32_t length_; int32_t num_patches_; @@ -595,6 +650,8 @@ class DecodeOperation : public TensorOperation { Status ValidateParams() override; + std::string Name() const override { return kDecodeOperation; } + private: bool rgb_; }; @@ -607,6 +664,8 @@ class EqualizeOperation : public TensorOperation { std::shared_ptr Build() override; Status ValidateParams() override; + + std::string Name() const override { return kEqualizeOperation; } }; class HwcToChwOperation : public TensorOperation { @@ -616,6 +675,8 @@ class HwcToChwOperation : public TensorOperation { std::shared_ptr Build() override; Status ValidateParams() override; + + std::string Name() const override { return kHwcToChwOperation; } }; class InvertOperation : public TensorOperation { @@ -625,6 +686,8 @@ class InvertOperation : public TensorOperation { std::shared_ptr Build() override; Status ValidateParams() override; + + std::string Name() const override { return kInvertOperation; } }; class MixUpBatchOperation : public TensorOperation { @@ -637,6 +700,8 @@ class MixUpBatchOperation : public TensorOperation { Status ValidateParams() override; + std::string Name() const override { return kMixUpBatchOperation; } + private: float alpha_; }; @@ -652,6 +717,8 @@ class NormalizeOperation : public TensorOperation { Status ValidateParams() override; + std::string Name() const override { return kNormalizeOperation; } + private: std::vector mean_; std::vector std_; @@ -669,6 +736,8 @@ class PadOperation : public TensorOperation { Status ValidateParams() override; + std::string Name() const override { return kPadOperation; } + private: std::vector padding_; std::vector fill_value_; @@ -689,6 +758,8 @@ class RandomAffineOperation : public TensorOperation { Status ValidateParams() override; + std::string Name() const override { return kRandomAffineOperation; } + private: std::vector degrees_; // min_degree, max_degree std::vector translate_range_; // maximum x translation percentage, maximum y translation percentage @@ -708,6 +779,8 @@ class RandomColorOperation : public TensorOperation { Status ValidateParams() override; + std::string Name() const override { return kRandomColorOperation; } + private: float t_lb_; float t_ub_; @@ -724,6 +797,8 @@ class RandomColorAdjustOperation : public TensorOperation { Status ValidateParams() override; + std::string Name() const override { return kRandomColorAdjustOperation; } + private: std::vector brightness_; std::vector contrast_; @@ -743,6 +818,8 @@ class RandomCropOperation : public TensorOperation { Status ValidateParams() override; + std::string Name() const override { return kRandomCropOperation; } + private: std::vector size_; std::vector padding_; @@ -762,6 +839,8 @@ class RandomCropDecodeResizeOperation : public TensorOperation { Status ValidateParams() override; + std::string Name() const override { return kRandomCropDecodeResizeOperation; } + private: std::vector size_; std::vector scale_; @@ -782,6 +861,8 @@ class RandomCropWithBBoxOperation : public TensorOperation { Status ValidateParams() override; + std::string Name() const override { return kRandomCropWithBBoxOperation; } + private: std::vector size_; std::vector padding_; @@ -800,6 +881,8 @@ class RandomHorizontalFlipOperation : public TensorOperation { Status ValidateParams() override; + std::string Name() const override { return kRandomHorizontalFlipOperation; } + private: float probability_; }; @@ -814,6 +897,8 @@ class RandomHorizontalFlipWithBBoxOperation : public TensorOperation { Status ValidateParams() override; + std::string Name() const override { return kRandomHorizontalFlipWithBBoxOperation; } + private: float probability_; }; @@ -828,6 +913,8 @@ class RandomPosterizeOperation : public TensorOperation { Status ValidateParams() override; + std::string Name() const override { return kRandomPosterizeOperation; } + private: std::vector bit_range_; }; @@ -842,6 +929,8 @@ class RandomResizeOperation : public TensorOperation { Status ValidateParams() override; + std::string Name() const override { return kRandomResizeOperation; } + private: std::vector size_; }; @@ -856,6 +945,8 @@ class RandomResizeWithBBoxOperation : public TensorOperation { Status ValidateParams() override; + std::string Name() const override { return kRandomResizeWithBBoxOperation; } + private: std::vector size_; }; @@ -873,6 +964,8 @@ class RandomResizedCropOperation : public TensorOperation { Status ValidateParams() override; + std::string Name() const override { return kRandomResizedCropOperation; } + private: std::vector size_; std::vector scale_; @@ -894,6 +987,8 @@ class RandomResizedCropWithBBoxOperation : public TensorOperation { Status ValidateParams() override; + std::string Name() const override { return kRandomResizedCropWithBBoxOperation; } + private: std::vector size_; std::vector scale_; @@ -913,6 +1008,8 @@ class RandomRotationOperation : public TensorOperation { Status ValidateParams() override; + std::string Name() const override { return kRandomRotationOperation; } + private: std::vector degrees_; InterpolationMode interpolation_mode_; @@ -932,6 +1029,8 @@ class RandomSelectSubpolicyOperation : public TensorOperation { Status ValidateParams() override; + std::string Name() const override { return kRandomSelectSubpolicyOperation; } + private: std::vector, double>>> policy_; }; @@ -946,6 +1045,8 @@ class RandomSharpnessOperation : public TensorOperation { Status ValidateParams() override; + std::string Name() const override { return kRandomSharpnessOperation; } + private: std::vector degrees_; }; @@ -960,6 +1061,8 @@ class RandomSolarizeOperation : public TensorOperation { Status ValidateParams() override; + std::string Name() const override { return kRandomSolarizeOperation; } + private: std::vector threshold_; }; @@ -974,6 +1077,8 @@ class RandomVerticalFlipOperation : public TensorOperation { Status ValidateParams() override; + std::string Name() const override { return kRandomVerticalFlipOperation; } + private: float probability_; }; @@ -988,6 +1093,8 @@ class RandomVerticalFlipWithBBoxOperation : public TensorOperation { Status ValidateParams() override; + std::string Name() const override { return kRandomVerticalFlipWithBBoxOperation; } + private: float probability_; }; @@ -1002,6 +1109,8 @@ class RescaleOperation : public TensorOperation { Status ValidateParams() override; + std::string Name() const override { return kRescaleOperation; } + private: float rescale_; float shift_; @@ -1019,6 +1128,8 @@ class ResizeOperation : public TensorOperation { Status ValidateParams() override; + std::string Name() const override { return kResizeOperation; } + private: std::vector size_; InterpolationMode interpolation_; @@ -1036,6 +1147,8 @@ class ResizeWithBBoxOperation : public TensorOperation { Status ValidateParams() override; + std::string Name() const override { return kResizeWithBBoxOperation; } + private: std::vector size_; InterpolationMode interpolation_; @@ -1050,6 +1163,8 @@ class RgbaToBgrOperation : public TensorOperation { std::shared_ptr Build() override; Status ValidateParams() override; + + std::string Name() const override { return kRgbaToBgrOperation; } }; class RgbaToRgbOperation : public TensorOperation { @@ -1061,6 +1176,8 @@ class RgbaToRgbOperation : public TensorOperation { std::shared_ptr Build() override; Status ValidateParams() override; + + std::string Name() const override { return kRgbaToRgbOperation; } }; class SoftDvppDecodeRandomCropResizeJpegOperation : public TensorOperation { @@ -1074,6 +1191,8 @@ class SoftDvppDecodeRandomCropResizeJpegOperation : public TensorOperation { Status ValidateParams() override; + std::string Name() const override { return kSoftDvppDecodeRandomCropResizeJpegOperation; } + private: std::vector size_; std::vector scale_; @@ -1091,6 +1210,8 @@ class SoftDvppDecodeResizeJpegOperation : public TensorOperation { Status ValidateParams() override; + std::string Name() const override { return kSoftDvppDecodeResizeJpegOperation; } + private: std::vector size_; }; @@ -1104,6 +1225,8 @@ class SwapRedBlueOperation : public TensorOperation { std::shared_ptr Build() override; Status ValidateParams() override; + + std::string Name() const override { return kSwapRedBlueOperation; } }; class UniformAugOperation : public TensorOperation { @@ -1116,6 +1239,8 @@ class UniformAugOperation : public TensorOperation { Status ValidateParams() override; + std::string Name() const override { return kUniformAugOperation; } + private: std::vector> transforms_; int32_t num_ops_; diff --git a/tests/ut/cpp/dataset/c_api_text_test.cc b/tests/ut/cpp/dataset/c_api_text_test.cc index 491eb92032..5a52bf0a22 100644 --- a/tests/ut/cpp/dataset/c_api_text_test.cc +++ b/tests/ut/cpp/dataset/c_api_text_test.cc @@ -461,6 +461,17 @@ TEST_F(MindDataTestPipeline, TestNgramFail) { EXPECT_EQ(ngram_op4, nullptr); } +TEST_F(MindDataTestPipeline, TestTextOperationName) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTextOperationName."; + + // Create object for the tensor op, and check the name + std::string data_file = datasets_root_path_ + "/testVocab/words.txt"; + std::shared_ptr sentence_piece_tokenizer_op = + text::SentencePieceTokenizer(data_file, SPieceTokenizerOutType::kString); + std::string correct_name = "SentencepieceTokenizer"; + EXPECT_EQ(correct_name, sentence_piece_tokenizer_op->Name()); +} + TEST_F(MindDataTestPipeline, TestWhitespaceTokenizerSuccess) { // Testing the parameter of WhitespaceTokenizer interface when the with_offsets is default. MS_LOG(INFO) << "Doing MindDataTestPipeline-TestWhitespaceTokenizerSuccess."; diff --git a/tests/ut/cpp/dataset/c_api_transforms_test.cc b/tests/ut/cpp/dataset/c_api_transforms_test.cc index 0719d386e5..e5d0ff3925 100644 --- a/tests/ut/cpp/dataset/c_api_transforms_test.cc +++ b/tests/ut/cpp/dataset/c_api_transforms_test.cc @@ -379,6 +379,15 @@ TEST_F(MindDataTestPipeline, TestRandomChoiceFail) { EXPECT_EQ(random_choice3, nullptr); } +TEST_F(MindDataTestPipeline, TestTransformOperationName) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTransformOperationName."; + + // Create object for the tensor op, and check the name + std::shared_ptr duplicate_op = transforms::Duplicate(); + std::string correct_name = "Duplicate"; + EXPECT_EQ(correct_name, duplicate_op->Name()); +} + TEST_F(MindDataTestPipeline, TestTypeCastSuccess) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTypeCastSuccess."; diff --git a/tests/ut/cpp/dataset/c_api_vision_test.cc b/tests/ut/cpp/dataset/c_api_vision_test.cc index e3dffab56e..77248a735b 100644 --- a/tests/ut/cpp/dataset/c_api_vision_test.cc +++ b/tests/ut/cpp/dataset/c_api_vision_test.cc @@ -3018,3 +3018,19 @@ TEST_F(MindDataTestPipeline, TestUniformAugWithOps) { // Manually terminate the pipeline iter->Stop(); } + +TEST_F(MindDataTestPipeline, TestVisionOperationName) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestVisionOperationName."; + + std::string correct_name; + + // Create object for the tensor op, and check the name + std::shared_ptr random_vertical_flip_op = vision::RandomVerticalFlip(0.5); + correct_name = "RandomVerticalFlip"; + EXPECT_EQ(correct_name, random_vertical_flip_op->Name()); + + // Create object for the tensor op, and check the name + std::shared_ptr softDvpp_decode_resize_jpeg_op = vision::SoftDvppDecodeResizeJpeg({1, 1}); + correct_name = "SoftDvppDecodeResizeJpeg"; + EXPECT_EQ(correct_name, softDvpp_decode_resize_jpeg_op->Name()); +}