diff --git a/mindspore/ccsrc/minddata/dataset/api/datasets.cc b/mindspore/ccsrc/minddata/dataset/api/datasets.cc index 03749a8668..ef3c8b8b07 100644 --- a/mindspore/ccsrc/minddata/dataset/api/datasets.cc +++ b/mindspore/ccsrc/minddata/dataset/api/datasets.cc @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -285,7 +285,24 @@ std::shared_ptr Schema(const std::string &schema_file) { // Function to create a AlbumDataset. std::shared_ptr Album(const std::string &dataset_dir, const std::string &data_schema, const std::vector &column_names, bool decode, - const std::shared_ptr &sampler, + const std::shared_ptr &sampler, + const std::shared_ptr &cache) { + auto ds = std::make_shared(dataset_dir, data_schema, column_names, decode, sampler, cache); + + return ds; +} +// Function to create a AlbumDataset. +std::shared_ptr Album(const std::string &dataset_dir, const std::string &data_schema, + const std::vector &column_names, bool decode, Sampler *sampler, + const std::shared_ptr &cache) { + auto ds = std::make_shared(dataset_dir, data_schema, column_names, decode, sampler, cache); + + return ds; +} +// Function to create a AlbumDataset. +std::shared_ptr Album(const std::string &dataset_dir, const std::string &data_schema, + const std::vector &column_names, bool decode, + const std::reference_wrapper sampler, const std::shared_ptr &cache) { auto ds = std::make_shared(dataset_dir, data_schema, column_names, decode, sampler, cache); @@ -295,7 +312,24 @@ std::shared_ptr Album(const std::string &dataset_dir, const std::s #ifndef ENABLE_ANDROID // Function to create a CelebADataset. std::shared_ptr CelebA(const std::string &dataset_dir, const std::string &usage, - const std::shared_ptr &sampler, bool decode, + const std::shared_ptr &sampler, bool decode, + const std::set &extensions, + const std::shared_ptr &cache) { + auto ds = std::make_shared(dataset_dir, usage, sampler, decode, extensions, cache); + + return ds; +} +// Function to create a CelebADataset. +std::shared_ptr CelebA(const std::string &dataset_dir, const std::string &usage, Sampler *sampler, + bool decode, const std::set &extensions, + const std::shared_ptr &cache) { + auto ds = std::make_shared(dataset_dir, usage, sampler, decode, extensions, cache); + + return ds; +} +// Function to create a CelebADataset. +std::shared_ptr CelebA(const std::string &dataset_dir, const std::string &usage, + const std::reference_wrapper sampler, bool decode, const std::set &extensions, const std::shared_ptr &cache) { auto ds = std::make_shared(dataset_dir, usage, sampler, decode, extensions, cache); @@ -305,7 +339,22 @@ std::shared_ptr CelebA(const std::string &dataset_dir, const std: // Function to create a Cifar10Dataset. std::shared_ptr Cifar10(const std::string &dataset_dir, const std::string &usage, - const std::shared_ptr &sampler, + const std::shared_ptr &sampler, + const std::shared_ptr &cache) { + auto ds = std::make_shared(dataset_dir, usage, sampler, cache); + + return ds; +} +// Function to create a Cifar10Dataset. +std::shared_ptr Cifar10(const std::string &dataset_dir, const std::string &usage, Sampler *sampler, + const std::shared_ptr &cache) { + auto ds = std::make_shared(dataset_dir, usage, sampler, cache); + + return ds; +} +// Function to create a Cifar10Dataset. +std::shared_ptr Cifar10(const std::string &dataset_dir, const std::string &usage, + const std::reference_wrapper sampler, const std::shared_ptr &cache) { auto ds = std::make_shared(dataset_dir, usage, sampler, cache); @@ -314,7 +363,22 @@ std::shared_ptr Cifar10(const std::string &dataset_dir, const st // Function to create a Cifar100Dataset. std::shared_ptr Cifar100(const std::string &dataset_dir, const std::string &usage, - const std::shared_ptr &sampler, + const std::shared_ptr &sampler, + const std::shared_ptr &cache) { + auto ds = std::make_shared(dataset_dir, usage, sampler, cache); + + return ds; +} +// Function to create a Cifar100Dataset. +std::shared_ptr Cifar100(const std::string &dataset_dir, const std::string &usage, Sampler *sampler, + const std::shared_ptr &cache) { + auto ds = std::make_shared(dataset_dir, usage, sampler, cache); + + return ds; +} +// Function to create a Cifar100Dataset. +std::shared_ptr Cifar100(const std::string &dataset_dir, const std::string &usage, + const std::reference_wrapper sampler, const std::shared_ptr &cache) { auto ds = std::make_shared(dataset_dir, usage, sampler, cache); @@ -330,10 +394,26 @@ std::shared_ptr CLUE(const std::vector &clue_files, co return ds; } +// Function to create a CocoDataset. +std::shared_ptr Coco(const std::string &dataset_dir, const std::string &annotation_file, + const std::string &task, const bool &decode, const std::shared_ptr &sampler, + const std::shared_ptr &cache) { + auto ds = std::make_shared(dataset_dir, annotation_file, task, decode, sampler, cache); + + return ds; +} +// Function to create a CocoDataset. +std::shared_ptr Coco(const std::string &dataset_dir, const std::string &annotation_file, + const std::string &task, const bool &decode, Sampler *sampler, + const std::shared_ptr &cache) { + auto ds = std::make_shared(dataset_dir, annotation_file, task, decode, sampler, cache); + + return ds; +} // Function to create a CocoDataset. std::shared_ptr Coco(const std::string &dataset_dir, const std::string &annotation_file, const std::string &task, const bool &decode, - const std::shared_ptr &sampler, + const std::reference_wrapper sampler, const std::shared_ptr &cache) { auto ds = std::make_shared(dataset_dir, annotation_file, task, decode, sampler, cache); @@ -353,7 +433,16 @@ std::shared_ptr CSV(const std::vector &dataset_files, c // Function to create a ImageFolderDataset. std::shared_ptr ImageFolder(const std::string &dataset_dir, bool decode, - const std::shared_ptr &sampler, + const std::shared_ptr &sampler, + const std::set &extensions, + const std::map &class_indexing, + const std::shared_ptr &cache) { + auto ds = std::make_shared(dataset_dir, decode, sampler, extensions, class_indexing, cache); + + return ds; +} +// Function to create a ImageFolderDataset. +std::shared_ptr ImageFolder(const std::string &dataset_dir, bool decode, Sampler *sampler, const std::set &extensions, const std::map &class_indexing, const std::shared_ptr &cache) { @@ -361,10 +450,37 @@ std::shared_ptr ImageFolder(const std::string &dataset_dir, return ds; } +// Function to create a ImageFolderDataset. +std::shared_ptr ImageFolder(const std::string &dataset_dir, bool decode, + const std::reference_wrapper sampler, + const std::set &extensions, + const std::map &class_indexing, + const std::shared_ptr &cache) { + auto ds = std::make_shared(dataset_dir, decode, sampler, extensions, class_indexing, cache); + + return ds; +} + +// Function to create a ManifestDataset. +std::shared_ptr Manifest(const std::string &dataset_file, const std::string &usage, + const std::shared_ptr &sampler, + const std::map &class_indexing, bool decode, + const std::shared_ptr &cache) { + auto ds = std::make_shared(dataset_file, usage, sampler, class_indexing, decode, cache); + + return ds; +} +// Function to create a ManifestDataset. +std::shared_ptr Manifest(const std::string &dataset_file, const std::string &usage, Sampler *sampler, + const std::map &class_indexing, bool decode, + const std::shared_ptr &cache) { + auto ds = std::make_shared(dataset_file, usage, sampler, class_indexing, decode, cache); + return ds; +} // Function to create a ManifestDataset. std::shared_ptr Manifest(const std::string &dataset_file, const std::string &usage, - const std::shared_ptr &sampler, + const std::reference_wrapper sampler, const std::map &class_indexing, bool decode, const std::shared_ptr &cache) { auto ds = std::make_shared(dataset_file, usage, sampler, class_indexing, decode, cache); @@ -374,7 +490,22 @@ std::shared_ptr Manifest(const std::string &dataset_file, const // Function to create a MindDataDataset. std::shared_ptr MindData(const std::string &dataset_file, const std::vector &columns_list, - const std::shared_ptr &sampler, nlohmann::json padded_sample, + const std::shared_ptr &sampler, nlohmann::json padded_sample, + int64_t num_padded) { + auto ds = std::make_shared(dataset_file, columns_list, sampler, padded_sample, num_padded); + + return ds; +} +// Function to create a MindDataDataset. +std::shared_ptr MindData(const std::string &dataset_file, const std::vector &columns_list, + Sampler *sampler, nlohmann::json padded_sample, int64_t num_padded) { + auto ds = std::make_shared(dataset_file, columns_list, sampler, padded_sample, num_padded); + + return ds; +} +// Function to create a MindDataDataset. +std::shared_ptr MindData(const std::string &dataset_file, const std::vector &columns_list, + const std::reference_wrapper sampler, nlohmann::json padded_sample, int64_t num_padded) { auto ds = std::make_shared(dataset_file, columns_list, sampler, padded_sample, num_padded); @@ -384,7 +515,24 @@ std::shared_ptr MindData(const std::string &dataset_file, const // Function to create a MindDataDataset. std::shared_ptr MindData(const std::vector &dataset_files, const std::vector &columns_list, - const std::shared_ptr &sampler, nlohmann::json padded_sample, + const std::shared_ptr &sampler, nlohmann::json padded_sample, + int64_t num_padded) { + auto ds = std::make_shared(dataset_files, columns_list, sampler, padded_sample, num_padded); + + return ds; +} +// Function to create a MindDataDataset. +std::shared_ptr MindData(const std::vector &dataset_files, + const std::vector &columns_list, Sampler *sampler, + nlohmann::json padded_sample, int64_t num_padded) { + auto ds = std::make_shared(dataset_files, columns_list, sampler, padded_sample, num_padded); + + return ds; +} +// Function to create a MindDataDataset. +std::shared_ptr MindData(const std::vector &dataset_files, + const std::vector &columns_list, + const std::reference_wrapper sampler, nlohmann::json padded_sample, int64_t num_padded) { auto ds = std::make_shared(dataset_files, columns_list, sampler, padded_sample, num_padded); @@ -393,7 +541,22 @@ std::shared_ptr MindData(const std::vector &datase #endif // Function to create a MnistDataset. std::shared_ptr Mnist(const std::string &dataset_dir, const std::string &usage, - const std::shared_ptr &sampler, + const std::shared_ptr &sampler, + const std::shared_ptr &cache) { + auto ds = std::make_shared(dataset_dir, usage, sampler, cache); + + return ds; +} +// Function to create a MnistDataset. +std::shared_ptr Mnist(const std::string &dataset_dir, const std::string &usage, Sampler *sampler, + const std::shared_ptr &cache) { + auto ds = std::make_shared(dataset_dir, usage, sampler, cache); + + return ds; +} +// Function to create a MnistDataset. +std::shared_ptr Mnist(const std::string &dataset_dir, const std::string &usage, + const std::reference_wrapper sampler, const std::shared_ptr &cache) { auto ds = std::make_shared(dataset_dir, usage, sampler, cache); @@ -419,7 +582,23 @@ std::shared_ptr TextFile(const std::vector &datase // Function to create a VOCDataset. std::shared_ptr VOC(const std::string &dataset_dir, const std::string &task, const std::string &usage, const std::map &class_indexing, bool decode, - const std::shared_ptr &sampler, + const std::shared_ptr &sampler, const std::shared_ptr &cache) { + auto ds = std::make_shared(dataset_dir, task, usage, class_indexing, decode, sampler, cache); + + return ds; +} +// Function to create a VOCDataset. +std::shared_ptr VOC(const std::string &dataset_dir, const std::string &task, const std::string &usage, + const std::map &class_indexing, bool decode, Sampler *sampler, + const std::shared_ptr &cache) { + auto ds = std::make_shared(dataset_dir, task, usage, class_indexing, decode, sampler, cache); + + return ds; +} +// Function to create a VOCDataset. +std::shared_ptr VOC(const std::string &dataset_dir, const std::string &task, const std::string &usage, + const std::map &class_indexing, bool decode, + const std::reference_wrapper sampler, const std::shared_ptr &cache) { auto ds = std::make_shared(dataset_dir, task, usage, class_indexing, decode, sampler, cache); @@ -831,41 +1010,117 @@ std::shared_ptr CreateDatasetCache(session_id_type id, uint64_t me AlbumDataset::AlbumDataset(const std::string &dataset_dir, const std::string &data_schema, const std::vector &column_names, bool decode, - const std::shared_ptr &sampler, const std::shared_ptr &cache) { - auto ds = std::make_shared(dataset_dir, data_schema, column_names, decode, sampler, cache); + const std::shared_ptr &sampler, const std::shared_ptr &cache) { + auto sampler_obj = sampler ? sampler->Parse() : nullptr; + auto ds = std::make_shared(dataset_dir, data_schema, column_names, decode, sampler_obj, cache); + ir_node_ = std::static_pointer_cast(ds); +} + +AlbumDataset::AlbumDataset(const std::string &dataset_dir, const std::string &data_schema, + const std::vector &column_names, bool decode, Sampler *sampler, + const std::shared_ptr &cache) { + auto sampler_obj = sampler ? sampler->Parse() : nullptr; + auto ds = std::make_shared(dataset_dir, data_schema, column_names, decode, sampler_obj, cache); + ir_node_ = std::static_pointer_cast(ds); +} +AlbumDataset::AlbumDataset(const std::string &dataset_dir, const std::string &data_schema, + const std::vector &column_names, bool decode, + const std::reference_wrapper sampler, const std::shared_ptr &cache) { + auto sampler_obj = sampler.get().Parse(); + auto ds = std::make_shared(dataset_dir, data_schema, column_names, decode, sampler_obj, cache); ir_node_ = std::static_pointer_cast(ds); } #ifndef ENABLE_ANDROID CelebADataset::CelebADataset(const std::string &dataset_dir, const std::string &usage, - const std::shared_ptr &sampler, bool decode, + const std::shared_ptr &sampler, bool decode, + const std::set &extensions, const std::shared_ptr &cache) { + auto sampler_obj = sampler ? sampler->Parse() : nullptr; + auto ds = std::make_shared(dataset_dir, usage, sampler_obj, decode, extensions, cache); + ir_node_ = std::static_pointer_cast(ds); +} +CelebADataset::CelebADataset(const std::string &dataset_dir, const std::string &usage, Sampler *sampler, bool decode, + const std::set &extensions, const std::shared_ptr &cache) { + auto sampler_obj = sampler ? sampler->Parse() : nullptr; + auto ds = std::make_shared(dataset_dir, usage, sampler_obj, decode, extensions, cache); + ir_node_ = std::static_pointer_cast(ds); +} +CelebADataset::CelebADataset(const std::string &dataset_dir, const std::string &usage, + const std::reference_wrapper sampler, bool decode, const std::set &extensions, const std::shared_ptr &cache) { - auto ds = std::make_shared(dataset_dir, usage, sampler, decode, extensions, cache); + auto sampler_obj = sampler.get().Parse(); + auto ds = std::make_shared(dataset_dir, usage, sampler_obj, decode, extensions, cache); ir_node_ = std::static_pointer_cast(ds); } + Cifar10Dataset::Cifar10Dataset(const std::string &dataset_dir, const std::string &usage, - const std::shared_ptr &sampler, const std::shared_ptr &cache) { - auto ds = std::make_shared(dataset_dir, usage, sampler, cache); + const std::shared_ptr &sampler, const std::shared_ptr &cache) { + auto sampler_obj = sampler ? sampler->Parse() : nullptr; + auto ds = std::make_shared(dataset_dir, usage, sampler_obj, cache); ir_node_ = std::static_pointer_cast(ds); } +Cifar10Dataset::Cifar10Dataset(const std::string &dataset_dir, const std::string &usage, Sampler *sampler, + const std::shared_ptr &cache) { + auto sampler_obj = sampler ? sampler->Parse() : nullptr; + auto ds = std::make_shared(dataset_dir, usage, sampler_obj, cache); + ir_node_ = std::static_pointer_cast(ds); +} +Cifar10Dataset::Cifar10Dataset(const std::string &dataset_dir, const std::string &usage, + const std::reference_wrapper sampler, + const std::shared_ptr &cache) { + auto sampler_obj = sampler.get().Parse(); + auto ds = std::make_shared(dataset_dir, usage, sampler_obj, cache); + ir_node_ = std::static_pointer_cast(ds); +} + Cifar100Dataset::Cifar100Dataset(const std::string &dataset_dir, const std::string &usage, - const std::shared_ptr &sampler, + const std::shared_ptr &sampler, const std::shared_ptr &cache) { + auto sampler_obj = sampler ? sampler->Parse() : nullptr; + auto ds = std::make_shared(dataset_dir, usage, sampler_obj, cache); + ir_node_ = std::static_pointer_cast(ds); +} +Cifar100Dataset::Cifar100Dataset(const std::string &dataset_dir, const std::string &usage, Sampler *sampler, const std::shared_ptr &cache) { - auto ds = std::make_shared(dataset_dir, usage, sampler, cache); + auto sampler_obj = sampler ? sampler->Parse() : nullptr; + auto ds = std::make_shared(dataset_dir, usage, sampler_obj, cache); ir_node_ = std::static_pointer_cast(ds); } +Cifar100Dataset::Cifar100Dataset(const std::string &dataset_dir, const std::string &usage, + const std::reference_wrapper sampler, + const std::shared_ptr &cache) { + auto sampler_obj = sampler.get().Parse(); + auto ds = std::make_shared(dataset_dir, usage, sampler_obj, cache); + ir_node_ = std::static_pointer_cast(ds); +} + CLUEDataset::CLUEDataset(const std::vector &dataset_files, const std::string &task, const std::string &usage, int64_t num_samples, ShuffleMode shuffle, int32_t num_shards, int32_t shard_id, const std::shared_ptr &cache) { auto ds = std::make_shared(dataset_files, task, usage, num_samples, shuffle, num_shards, shard_id, cache); ir_node_ = std::static_pointer_cast(ds); } + +CocoDataset::CocoDataset(const std::string &dataset_dir, const std::string &annotation_file, const std::string &task, + const bool &decode, const std::shared_ptr &sampler, + const std::shared_ptr &cache) { + auto sampler_obj = sampler ? sampler->Parse() : nullptr; + auto ds = std::make_shared(dataset_dir, annotation_file, task, decode, sampler_obj, cache); + ir_node_ = std::static_pointer_cast(ds); +} +CocoDataset::CocoDataset(const std::string &dataset_dir, const std::string &annotation_file, const std::string &task, + const bool &decode, Sampler *sampler, const std::shared_ptr &cache) { + auto sampler_obj = sampler ? sampler->Parse() : nullptr; + auto ds = std::make_shared(dataset_dir, annotation_file, task, decode, sampler_obj, cache); + ir_node_ = std::static_pointer_cast(ds); +} CocoDataset::CocoDataset(const std::string &dataset_dir, const std::string &annotation_file, const std::string &task, - const bool &decode, const std::shared_ptr &sampler, + const bool &decode, const std::reference_wrapper sampler, const std::shared_ptr &cache) { - auto ds = std::make_shared(dataset_dir, annotation_file, task, decode, sampler, cache); + auto sampler_obj = sampler.get().Parse(); + auto ds = std::make_shared(dataset_dir, annotation_file, task, decode, sampler_obj, cache); ir_node_ = std::static_pointer_cast(ds); } + CSVDataset::CSVDataset(const std::vector &dataset_files, char field_delim, const std::vector> &column_defaults, const std::vector &column_names, int64_t num_samples, ShuffleMode shuffle, @@ -874,8 +1129,21 @@ CSVDataset::CSVDataset(const std::vector &dataset_files, char field num_shards, shard_id, cache); ir_node_ = std::static_pointer_cast(ds); } + ImageFolderDataset::ImageFolderDataset(const std::string &dataset_dir, bool decode, - const std::shared_ptr &sampler, + const std::shared_ptr &sampler, const std::set &extensions, + const std::map &class_indexing, + const std::shared_ptr &cache) { + // This arg exists in ImageFolderOp, but not externalized (in Python API). The default value is false. + bool recursive = false; + + // Create logical representation of ImageFolderDataset. + auto sampler_obj = sampler ? sampler->Parse() : nullptr; + auto ds = + std::make_shared(dataset_dir, decode, sampler_obj, recursive, extensions, class_indexing, cache); + ir_node_ = std::static_pointer_cast(ds); +} +ImageFolderDataset::ImageFolderDataset(const std::string &dataset_dir, bool decode, Sampler *sampler, const std::set &extensions, const std::map &class_indexing, const std::shared_ptr &cache) { @@ -883,36 +1151,110 @@ ImageFolderDataset::ImageFolderDataset(const std::string &dataset_dir, bool deco bool recursive = false; // Create logical representation of ImageFolderDataset. + auto sampler_obj = sampler ? sampler->Parse() : nullptr; auto ds = - std::make_shared(dataset_dir, decode, sampler, recursive, extensions, class_indexing, cache); + std::make_shared(dataset_dir, decode, sampler_obj, recursive, extensions, class_indexing, cache); ir_node_ = std::static_pointer_cast(ds); } +ImageFolderDataset::ImageFolderDataset(const std::string &dataset_dir, bool decode, + const std::reference_wrapper sampler, + const std::set &extensions, + const std::map &class_indexing, + const std::shared_ptr &cache) { + // This arg exists in ImageFolderOp, but not externalized (in Python API). The default value is false. + bool recursive = false; + // Create logical representation of ImageFolderDataset. + auto sampler_obj = sampler.get().Parse(); + auto ds = + std::make_shared(dataset_dir, decode, sampler_obj, recursive, extensions, class_indexing, cache); + ir_node_ = std::static_pointer_cast(ds); +} + +ManifestDataset::ManifestDataset(const std::string &dataset_file, const std::string &usage, + const std::shared_ptr &sampler, + const std::map &class_indexing, bool decode, + const std::shared_ptr &cache) { + auto sampler_obj = sampler ? sampler->Parse() : nullptr; + auto ds = std::make_shared(dataset_file, usage, sampler_obj, class_indexing, decode, cache); + ir_node_ = std::static_pointer_cast(ds); +} +ManifestDataset::ManifestDataset(const std::string &dataset_file, const std::string &usage, Sampler *sampler, + const std::map &class_indexing, bool decode, + const std::shared_ptr &cache) { + auto sampler_obj = sampler ? sampler->Parse() : nullptr; + auto ds = std::make_shared(dataset_file, usage, sampler_obj, class_indexing, decode, cache); + ir_node_ = std::static_pointer_cast(ds); +} ManifestDataset::ManifestDataset(const std::string &dataset_file, const std::string &usage, - const std::shared_ptr &sampler, + const std::reference_wrapper sampler, const std::map &class_indexing, bool decode, const std::shared_ptr &cache) { - auto ds = std::make_shared(dataset_file, usage, sampler, class_indexing, decode, cache); + auto sampler_obj = sampler.get().Parse(); + auto ds = std::make_shared(dataset_file, usage, sampler_obj, class_indexing, decode, cache); + ir_node_ = std::static_pointer_cast(ds); +} + +MindDataDataset::MindDataDataset(const std::string &dataset_file, const std::vector &columns_list, + const std::shared_ptr &sampler, nlohmann::json padded_sample, + int64_t num_padded) { + auto sampler_obj = sampler ? sampler->Parse() : nullptr; + auto ds = std::make_shared(dataset_file, columns_list, sampler_obj, padded_sample, num_padded); + ir_node_ = std::static_pointer_cast(ds); +} +MindDataDataset::MindDataDataset(const std::string &dataset_file, const std::vector &columns_list, + Sampler *sampler, nlohmann::json padded_sample, int64_t num_padded) { + auto sampler_obj = sampler ? sampler->Parse() : nullptr; + auto ds = std::make_shared(dataset_file, columns_list, sampler_obj, padded_sample, num_padded); ir_node_ = std::static_pointer_cast(ds); } MindDataDataset::MindDataDataset(const std::string &dataset_file, const std::vector &columns_list, - const std::shared_ptr &sampler, nlohmann::json padded_sample, + const std::reference_wrapper sampler, nlohmann::json padded_sample, int64_t num_padded) { - auto ds = std::make_shared(dataset_file, columns_list, sampler, padded_sample, num_padded); + auto sampler_obj = sampler.get().Parse(); + auto ds = std::make_shared(dataset_file, columns_list, sampler_obj, padded_sample, num_padded); + ir_node_ = std::static_pointer_cast(ds); +} +MindDataDataset::MindDataDataset(const std::vector &dataset_files, + const std::vector &columns_list, const std::shared_ptr &sampler, + nlohmann::json padded_sample, int64_t num_padded) { + auto sampler_obj = sampler ? sampler->Parse() : nullptr; + auto ds = std::make_shared(dataset_files, columns_list, sampler_obj, padded_sample, num_padded); + ir_node_ = std::static_pointer_cast(ds); +} +MindDataDataset::MindDataDataset(const std::vector &dataset_files, + const std::vector &columns_list, Sampler *sampler, + nlohmann::json padded_sample, int64_t num_padded) { + auto sampler_obj = sampler ? sampler->Parse() : nullptr; + auto ds = std::make_shared(dataset_files, columns_list, sampler_obj, padded_sample, num_padded); ir_node_ = std::static_pointer_cast(ds); } MindDataDataset::MindDataDataset(const std::vector &dataset_files, const std::vector &columns_list, - const std::shared_ptr &sampler, nlohmann::json padded_sample, + const std::reference_wrapper sampler, nlohmann::json padded_sample, int64_t num_padded) { - auto ds = std::make_shared(dataset_files, columns_list, sampler, padded_sample, num_padded); + auto sampler_obj = sampler.get().Parse(); + auto ds = std::make_shared(dataset_files, columns_list, sampler_obj, padded_sample, num_padded); ir_node_ = std::static_pointer_cast(ds); } #endif MnistDataset::MnistDataset(const std::string &dataset_dir, const std::string &usage, - const std::shared_ptr &sampler, const std::shared_ptr &cache) { - auto ds = std::make_shared(dataset_dir, usage, sampler, cache); + const std::shared_ptr &sampler, const std::shared_ptr &cache) { + auto sampler_obj = sampler ? sampler->Parse() : nullptr; + auto ds = std::make_shared(dataset_dir, usage, sampler_obj, cache); + ir_node_ = std::static_pointer_cast(ds); +} +MnistDataset::MnistDataset(const std::string &dataset_dir, const std::string &usage, Sampler *sampler, + const std::shared_ptr &cache) { + auto sampler_obj = sampler ? sampler->Parse() : nullptr; + auto ds = std::make_shared(dataset_dir, usage, sampler_obj, cache); + ir_node_ = std::static_pointer_cast(ds); +} +MnistDataset::MnistDataset(const std::string &dataset_dir, const std::string &usage, + const std::reference_wrapper sampler, const std::shared_ptr &cache) { + auto sampler_obj = sampler.get().Parse(); + auto ds = std::make_shared(dataset_dir, usage, sampler_obj, cache); ir_node_ = std::static_pointer_cast(ds); } @@ -926,8 +1268,23 @@ TextFileDataset::TextFileDataset(const std::vector &dataset_files, VOCDataset::VOCDataset(const std::string &dataset_dir, const std::string &task, const std::string &usage, const std::map &class_indexing, bool decode, - const std::shared_ptr &sampler, const std::shared_ptr &cache) { - auto ds = std::make_shared(dataset_dir, task, usage, class_indexing, decode, sampler, cache); + const std::shared_ptr &sampler, const std::shared_ptr &cache) { + auto sampler_obj = sampler ? sampler->Parse() : nullptr; + auto ds = std::make_shared(dataset_dir, task, usage, class_indexing, decode, sampler_obj, cache); + ir_node_ = std::static_pointer_cast(ds); +} +VOCDataset::VOCDataset(const std::string &dataset_dir, const std::string &task, const std::string &usage, + const std::map &class_indexing, bool decode, Sampler *sampler, + const std::shared_ptr &cache) { + auto sampler_obj = sampler ? sampler->Parse() : nullptr; + auto ds = std::make_shared(dataset_dir, task, usage, class_indexing, decode, sampler_obj, cache); + ir_node_ = std::static_pointer_cast(ds); +} +VOCDataset::VOCDataset(const std::string &dataset_dir, const std::string &task, const std::string &usage, + const std::map &class_indexing, bool decode, + const std::reference_wrapper sampler, const std::shared_ptr &cache) { + auto sampler_obj = sampler.get().Parse(); + auto ds = std::make_shared(dataset_dir, task, usage, class_indexing, decode, sampler_obj, cache); ir_node_ = std::static_pointer_cast(ds); } diff --git a/mindspore/ccsrc/minddata/dataset/api/samplers.cc b/mindspore/ccsrc/minddata/dataset/api/samplers.cc index b61b90397f..50046d135e 100644 --- a/mindspore/ccsrc/minddata/dataset/api/samplers.cc +++ b/mindspore/ccsrc/minddata/dataset/api/samplers.cc @@ -20,78 +20,68 @@ namespace mindspore { namespace dataset { -/// Function to create a Distributed Sampler. -std::shared_ptr DistributedSampler(int64_t num_shards, int64_t shard_id, bool shuffle, - int64_t num_samples, uint32_t seed, int64_t offset, - bool even_dist) { - auto sampler = - std::make_shared(num_shards, shard_id, shuffle, num_samples, seed, offset, even_dist); - // Input validation - if (sampler->ValidateParams().IsError()) { - return nullptr; - } - return sampler; +// DistributedSampler +DistributedSampler::DistributedSampler(int64_t num_shards, int64_t shard_id, bool shuffle, int64_t num_samples, + uint32_t seed, int64_t offset, bool even_dist) + : num_shards_(num_shards), + shard_id_(shard_id), + shuffle_(shuffle), + num_samples_(num_samples), + seed_(seed), + offset_(offset), + even_dist_(even_dist) {} + +std::shared_ptr DistributedSampler::Parse() { + return std::make_shared(num_shards_, shard_id_, shuffle_, num_samples_, seed_, offset_, + even_dist_); } -/// Function to create a PK Sampler. -std::shared_ptr PKSampler(int64_t num_val, bool shuffle, int64_t num_samples) { - auto sampler = std::make_shared(num_val, shuffle, num_samples); - // Input validation - if (sampler->ValidateParams().IsError()) { - return nullptr; - } - return sampler; +// PKSampler +PKSampler::PKSampler(int64_t num_val, bool shuffle, int64_t num_samples) + : num_val_(num_val), shuffle_(shuffle), num_samples_(num_samples) {} + +std::shared_ptr PKSampler::Parse() { + return std::make_shared(num_val_, shuffle_, num_samples_); } -/// Function to create a Random Sampler. -std::shared_ptr RandomSampler(bool replacement, int64_t num_samples) { - auto sampler = std::make_shared(replacement, num_samples); - // Input validation - if (sampler->ValidateParams().IsError()) { - return nullptr; - } - return sampler; +// RandomSampler +RandomSampler::RandomSampler(bool replacement, int64_t num_samples) + : replacement_(replacement), num_samples_(num_samples) {} + +std::shared_ptr RandomSampler::Parse() { + return std::make_shared(replacement_, num_samples_); } -/// Function to create a Sequential Sampler. -std::shared_ptr SequentialSampler(int64_t start_index, int64_t num_samples) { - auto sampler = std::make_shared(start_index, num_samples); - // Input validation - if (sampler->ValidateParams().IsError()) { - return nullptr; - } - return sampler; +// SequentialSampler +SequentialSampler::SequentialSampler(int64_t start_index, int64_t num_samples) + : start_index_(start_index), num_samples_(num_samples) {} + +std::shared_ptr SequentialSampler::Parse() { + return std::make_shared(start_index_, num_samples_); } -/// Function to create a Subset Random Sampler. -std::shared_ptr SubsetSampler(std::vector indices, int64_t num_samples) { - auto sampler = std::make_shared(std::move(indices), num_samples); - // Input validation - if (sampler->ValidateParams().IsError()) { - return nullptr; - } - return sampler; +// SubsetSampler +SubsetSampler::SubsetSampler(std::vector indices, int64_t num_samples) + : indices_(indices), num_samples_(num_samples) {} + +std::shared_ptr SubsetSampler::Parse() { + return std::make_shared(indices_, num_samples_); } -/// Function to create a Subset Random Sampler. -std::shared_ptr SubsetRandomSampler(std::vector indices, int64_t num_samples) { - auto sampler = std::make_shared(std::move(indices), num_samples); - // Input validation - if (sampler->ValidateParams().IsError()) { - return nullptr; - } - return sampler; +// SubsetRandomSampler +SubsetRandomSampler::SubsetRandomSampler(std::vector indices, int64_t num_samples) + : SubsetSampler(indices, num_samples) {} + +std::shared_ptr SubsetRandomSampler::Parse() { + return std::make_shared(indices_, num_samples_); } -/// Function to create a Weighted Random Sampler. -std::shared_ptr WeightedRandomSampler(std::vector weights, int64_t num_samples, - bool replacement) { - auto sampler = std::make_shared(std::move(weights), num_samples, replacement); - // Input validation - if (sampler->ValidateParams().IsError()) { - return nullptr; - } - return sampler; +// WeightedRandomSampler +WeightedRandomSampler::WeightedRandomSampler(std::vector weights, int64_t num_samples, bool replacement) + : weights_(weights), num_samples_(num_samples), replacement_(replacement) {} + +std::shared_ptr WeightedRandomSampler::Parse() { + return std::make_shared(weights_, num_samples_, replacement_); } } // namespace dataset diff --git a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/dataset_node.cc b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/dataset_node.cc index a3cba83fd6..cf370714ac 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/dataset_node.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/dataset_node.cc @@ -138,7 +138,7 @@ Status ValidateDatasetShardParams(const std::string &dataset_name, int32_t num_s // Helper function to validate dataset sampler parameter Status ValidateDatasetSampler(const std::string &dataset_name, const std::shared_ptr &sampler) { - if (sampler == nullptr) { + if (sampler == nullptr || sampler->ValidateParams().IsError()) { std::string err_msg = dataset_name + ": Sampler is not constructed correctly, sampler: nullptr"; MS_LOG(ERROR) << err_msg; RETURN_STATUS_SYNTAX_ERROR(err_msg); @@ -191,17 +191,17 @@ std::shared_ptr SelectSampler(int64_t num_samples, bool shuffle, int if (shuffle) { if (num_shards > 1) { // If shuffle enabled, sharding enabled, use distributed random sampler - return DistributedSampler(num_shards, shard_id, shuffle, num_samples); + return DistributedSampler(num_shards, shard_id, shuffle, num_samples).Parse(); } // If shuffle enabled, sharding disabled, use random sampler - return RandomSampler(num_samples >= 0, num_samples); + return RandomSampler(num_samples >= 0, num_samples).Parse(); } if (num_shards > 1) { // If shuffle disabled, sharding enabled, use distributed sequential sampler - return DistributedSampler(num_shards, shard_id, shuffle, num_samples); + return DistributedSampler(num_shards, shard_id, shuffle, num_samples).Parse(); } // If shuffle disabled, sharding disabled, use sequential sampler - return SequentialSampler(0, num_samples); + return SequentialSampler(0, num_samples).Parse(); } // Constructor to initialize the cache diff --git a/mindspore/ccsrc/minddata/dataset/include/datasets.h b/mindspore/ccsrc/minddata/dataset/include/datasets.h index ef50e9859b..4cfa0086ab 100644 --- a/mindspore/ccsrc/minddata/dataset/include/datasets.h +++ b/mindspore/ccsrc/minddata/dataset/include/datasets.h @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -585,7 +585,13 @@ class AlbumDataset : public Dataset { public: AlbumDataset(const std::string &dataset_dir, const std::string &data_schema, const std::vector &column_names = {}, bool decode = false, - const std::shared_ptr &sampler = RandomSampler(), + const std::shared_ptr &sampler = std::make_shared(), + const std::shared_ptr &cache = nullptr); + AlbumDataset(const std::string &dataset_dir, const std::string &data_schema, + const std::vector &column_names, bool decode, Sampler *sampler, + const std::shared_ptr &cache = nullptr); + AlbumDataset(const std::string &dataset_dir, const std::string &data_schema, + const std::vector &column_names, bool decode, const std::reference_wrapper sampler, const std::shared_ptr &cache = nullptr); ~AlbumDataset() = default; }; @@ -597,20 +603,53 @@ class AlbumDataset : public Dataset { /// \param[in] column_names Column names used to specify columns to load, if empty, will read all columns. /// (default = {}) /// \param[in] decode the option to decode the images in dataset (default = false) -/// \param[in] sampler Object used to choose samples from the dataset. If sampler is not given, +/// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not +/// given, /// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()) /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). /// \return Shared pointer to the current Dataset std::shared_ptr Album(const std::string &dataset_dir, const std::string &data_schema, const std::vector &column_names = {}, bool decode = false, - const std::shared_ptr &sampler = RandomSampler(), + const std::shared_ptr &sampler = std::make_shared(), + const std::shared_ptr &cache = nullptr); +/// \brief Function to create an AlbumDataset +/// \notes The generated dataset is specified through setting a schema +/// \param[in] dataset_dir Path to the root directory that contains the dataset +/// \param[in] data_schema Path to dataset schema file +/// \param[in] column_names Column names used to specify columns to load +/// \param[in] decode the option to decode the images in dataset +/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset. +/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). +/// \return Shared pointer to the current Dataset +std::shared_ptr Album(const std::string &dataset_dir, const std::string &data_schema, + const std::vector &column_names, bool decode, Sampler *sampler, + const std::shared_ptr &cache = nullptr); +/// \brief Function to create an AlbumDataset +/// \notes The generated dataset is specified through setting a schema +/// \param[in] dataset_dir Path to the root directory that contains the dataset +/// \param[in] data_schema Path to dataset schema file +/// \param[in] column_names Column names used to specify columns to load +/// \param[in] decode the option to decode the images in dataset +/// \param[in] sampler Sampler object used to choose samples from the dataset. +/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). +/// \return Shared pointer to the current Dataset +std::shared_ptr Album(const std::string &dataset_dir, const std::string &data_schema, + const std::vector &column_names, bool decode, + const std::reference_wrapper sampler, const std::shared_ptr &cache = nullptr); #ifndef ENABLE_ANDROID class CelebADataset : public Dataset { public: explicit CelebADataset(const std::string &dataset_dir, const std::string &usage = "all", - const std::shared_ptr &sampler = RandomSampler(), bool decode = false, + const std::shared_ptr &sampler = std::make_shared(), + bool decode = false, const std::set &extensions = {}, + const std::shared_ptr &cache = nullptr); + explicit CelebADataset(const std::string &dataset_dir, const std::string &usage, Sampler *sampler, + bool decode = false, const std::set &extensions = {}, + const std::shared_ptr &cache = nullptr); + explicit CelebADataset(const std::string &dataset_dir, const std::string &usage, + const std::reference_wrapper sampler, bool decode = false, const std::set &extensions = {}, const std::shared_ptr &cache = nullptr); ~CelebADataset() = default; @@ -621,21 +660,54 @@ class CelebADataset : public Dataset { /// The type of the image tensor is uint8. The attr tensor is uint32 and one hot type. /// \param[in] dataset_dir Path to the root directory that contains the dataset. /// \param[in] usage One of "all", "train", "valid" or "test" (default = "all"). -/// \param[in] sampler Object used to choose samples from the dataset. If sampler is not given, +/// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not +/// given, /// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()) /// \param[in] decode Decode the images after reading (default=false). /// \param[in] extensions Set of file extensions to be included in the dataset (default={}). /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). /// \return Shared pointer to the current Dataset std::shared_ptr CelebA(const std::string &dataset_dir, const std::string &usage = "all", - const std::shared_ptr &sampler = RandomSampler(), bool decode = false, + const std::shared_ptr &sampler = std::make_shared(), + bool decode = false, const std::set &extensions = {}, + const std::shared_ptr &cache = nullptr); +/// \brief Function to create a CelebADataset +/// \notes The generated dataset has two columns ['image', 'attr']. +/// The type of the image tensor is uint8. The attr tensor is uint32 and one hot type. +/// \param[in] dataset_dir Path to the root directory that contains the dataset. +/// \param[in] usage One of "all", "train", "valid" or "test" +/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset. +/// \param[in] decode Decode the images after reading (default=false). +/// \param[in] extensions Set of file extensions to be included in the dataset (default={}). +/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). +/// \return Shared pointer to the current Dataset +std::shared_ptr CelebA(const std::string &dataset_dir, const std::string &usage, Sampler *sampler, + bool decode = false, const std::set &extensions = {}, + const std::shared_ptr &cache = nullptr); +/// \brief Function to create a CelebADataset +/// \notes The generated dataset has two columns ['image', 'attr']. +/// The type of the image tensor is uint8. The attr tensor is uint32 and one hot type. +/// \param[in] dataset_dir Path to the root directory that contains the dataset. +/// \param[in] usage One of "all", "train", "valid" or "test" +/// \param[in] sampler Sampler object used to choose samples from the dataset. +/// \param[in] decode Decode the images after reading (default=false). +/// \param[in] extensions Set of file extensions to be included in the dataset (default={}). +/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). +/// \return Shared pointer to the current Dataset +std::shared_ptr CelebA(const std::string &dataset_dir, const std::string &usage, + const std::reference_wrapper sampler, bool decode = false, const std::set &extensions = {}, const std::shared_ptr &cache = nullptr); class Cifar10Dataset : public Dataset { public: explicit Cifar10Dataset(const std::string &dataset_dir, const std::string &usage = "all", - const std::shared_ptr &sampler = RandomSampler(), + const std::shared_ptr &sampler = std::make_shared(), + const std::shared_ptr &cache = nullptr); + explicit Cifar10Dataset(const std::string &dataset_dir, const std::string &usage, Sampler *sampler, + const std::shared_ptr &cache = nullptr); + explicit Cifar10Dataset(const std::string &dataset_dir, const std::string &usage, + const std::reference_wrapper sampler, const std::shared_ptr &cache = nullptr); ~Cifar10Dataset() = default; }; @@ -644,18 +716,43 @@ class Cifar10Dataset : public Dataset { /// \notes The generated dataset has two columns ["image", "label"] /// \param[in] dataset_dir Path to the root directory that contains the dataset /// \param[in] usage of CIFAR10, can be "train", "test" or "all" (default = "all"). -/// \param[in] sampler Object used to choose samples from the dataset. If sampler is not given, +/// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not +/// given, /// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()) /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). /// \return Shared pointer to the current Dataset std::shared_ptr Cifar10(const std::string &dataset_dir, const std::string &usage = "all", - const std::shared_ptr &sampler = RandomSampler(), + const std::shared_ptr &sampler = std::make_shared(), + const std::shared_ptr &cache = nullptr); +/// \brief Function to create a Cifar10 Dataset +/// \notes The generated dataset has two columns ["image", "label"] +/// \param[in] dataset_dir Path to the root directory that contains the dataset +/// \param[in] usage of CIFAR10, can be "train", "test" or "all" +/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset. +/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). +/// \return Shared pointer to the current Dataset +std::shared_ptr Cifar10(const std::string &dataset_dir, const std::string &usage, Sampler *sampler, + const std::shared_ptr &cache = nullptr); +/// \brief Function to create a Cifar10 Dataset +/// \notes The generated dataset has two columns ["image", "label"] +/// \param[in] dataset_dir Path to the root directory that contains the dataset +/// \param[in] usage of CIFAR10, can be "train", "test" or "all" +/// \param[in] sampler Sampler object used to choose samples from the dataset. +/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). +/// \return Shared pointer to the current Dataset +std::shared_ptr Cifar10(const std::string &dataset_dir, const std::string &usage, + const std::reference_wrapper sampler, const std::shared_ptr &cache = nullptr); class Cifar100Dataset : public Dataset { public: explicit Cifar100Dataset(const std::string &dataset_dir, const std::string &usage = "all", - const std::shared_ptr &sampler = RandomSampler(), + const std::shared_ptr &sampler = std::make_shared(), + const std::shared_ptr &cache = nullptr); + explicit Cifar100Dataset(const std::string &dataset_dir, const std::string &usage, Sampler *sampler, + const std::shared_ptr &cache = nullptr); + explicit Cifar100Dataset(const std::string &dataset_dir, const std::string &usage, + const std::reference_wrapper sampler, const std::shared_ptr &cache = nullptr); ~Cifar100Dataset() = default; }; @@ -664,12 +761,32 @@ class Cifar100Dataset : public Dataset { /// \notes The generated dataset has three columns ["image", "coarse_label", "fine_label"] /// \param[in] dataset_dir Path to the root directory that contains the dataset /// \param[in] usage of CIFAR100, can be "train", "test" or "all" (default = "all"). -/// \param[in] sampler Object used to choose samples from the dataset. If sampler is not given, +/// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not +/// given, /// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()) /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). /// \return Shared pointer to the current Dataset std::shared_ptr Cifar100(const std::string &dataset_dir, const std::string &usage = "all", - const std::shared_ptr &sampler = RandomSampler(), + const std::shared_ptr &sampler = std::make_shared(), + const std::shared_ptr &cache = nullptr); +/// \brief Function to create a Cifar100 Dataset +/// \notes The generated dataset has three columns ["image", "coarse_label", "fine_label"] +/// \param[in] dataset_dir Path to the root directory that contains the dataset +/// \param[in] usage of CIFAR100, can be "train", "test" or "all". +/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset. +/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). +/// \return Shared pointer to the current Dataset +std::shared_ptr Cifar100(const std::string &dataset_dir, const std::string &usage, Sampler *sampler, + const std::shared_ptr &cache = nullptr); +/// \brief Function to create a Cifar100 Dataset +/// \notes The generated dataset has three columns ["image", "coarse_label", "fine_label"] +/// \param[in] dataset_dir Path to the root directory that contains the dataset +/// \param[in] usage of CIFAR100, can be "train", "test" or "all". +/// \param[in] sampler Sampler object used to choose samples from the dataset. +/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). +/// \return Shared pointer to the current Dataset +std::shared_ptr Cifar100(const std::string &dataset_dir, const std::string &usage, + const std::reference_wrapper sampler, const std::shared_ptr &cache = nullptr); class CLUEDataset : public Dataset { @@ -707,7 +824,12 @@ std::shared_ptr CLUE(const std::vector &dataset_files, class CocoDataset : public Dataset { public: CocoDataset(const std::string &dataset_dir, const std::string &annotation_file, const std::string &task = "Detection", - const bool &decode = false, const std::shared_ptr &sampler = RandomSampler(), + const bool &decode = false, const std::shared_ptr &sampler = std::make_shared(), + const std::shared_ptr &cache = nullptr); + CocoDataset(const std::string &dataset_dir, const std::string &annotation_file, const std::string &task, + const bool &decode, Sampler *sampler, const std::shared_ptr &cache = nullptr); + CocoDataset(const std::string &dataset_dir, const std::string &annotation_file, const std::string &task, + const bool &decode, const std::reference_wrapper sampler, const std::shared_ptr &cache = nullptr); ~CocoDataset() = default; }; @@ -725,13 +847,53 @@ class CocoDataset : public Dataset { /// \param[in] annotation_file Path to the annotation json /// \param[in] task Set the task type of reading coco data, now support 'Detection'/'Stuff'/'Panoptic'/'Keypoint' /// \param[in] decode Decode the images after reading -/// \param[in] sampler Object used to choose samples from the dataset. If sampler is not given, +/// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not +/// given, /// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()) /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). /// \return Shared pointer to the current Dataset std::shared_ptr Coco(const std::string &dataset_dir, const std::string &annotation_file, const std::string &task = "Detection", const bool &decode = false, - const std::shared_ptr &sampler = RandomSampler(), + const std::shared_ptr &sampler = std::make_shared(), + const std::shared_ptr &cache = nullptr); +/// \brief Function to create a CocoDataset +/// \notes The generated dataset has multi-columns : +/// - task='Detection', column: [['image', dtype=uint8], ['bbox', dtype=float32], ['category_id', dtype=uint32], +/// ['iscrowd', dtype=uint32]]. +/// - task='Stuff', column: [['image', dtype=uint8], ['segmentation',dtype=float32], ['iscrowd', dtype=uint32]]. +/// - task='Keypoint', column: [['image', dtype=uint8], ['keypoints', dtype=float32], +/// ['num_keypoints', dtype=uint32]]. +/// - task='Panoptic', column: [['image', dtype=uint8], ['bbox', dtype=float32], ['category_id', dtype=uint32], +/// ['iscrowd', dtype=uint32], ['area', dtype=uitn32]]. +/// \param[in] dataset_dir Path to the root directory that contains the dataset +/// \param[in] annotation_file Path to the annotation json +/// \param[in] task Set the task type of reading coco data, now support 'Detection'/'Stuff'/'Panoptic'/'Keypoint' +/// \param[in] decode Decode the images after reading +/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset. +/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). +/// \return Shared pointer to the current Dataset +std::shared_ptr Coco(const std::string &dataset_dir, const std::string &annotation_file, + const std::string &task, const bool &decode, Sampler *sampler, + const std::shared_ptr &cache = nullptr); +/// \brief Function to create a CocoDataset +/// \notes The generated dataset has multi-columns : +/// - task='Detection', column: [['image', dtype=uint8], ['bbox', dtype=float32], ['category_id', dtype=uint32], +/// ['iscrowd', dtype=uint32]]. +/// - task='Stuff', column: [['image', dtype=uint8], ['segmentation',dtype=float32], ['iscrowd', dtype=uint32]]. +/// - task='Keypoint', column: [['image', dtype=uint8], ['keypoints', dtype=float32], +/// ['num_keypoints', dtype=uint32]]. +/// - task='Panoptic', column: [['image', dtype=uint8], ['bbox', dtype=float32], ['category_id', dtype=uint32], +/// ['iscrowd', dtype=uint32], ['area', dtype=uitn32]]. +/// \param[in] dataset_dir Path to the root directory that contains the dataset +/// \param[in] annotation_file Path to the annotation json +/// \param[in] task Set the task type of reading coco data, now support 'Detection'/'Stuff'/'Panoptic'/'Keypoint' +/// \param[in] decode Decode the images after reading +/// \param[in] sampler Sampler object used to choose samples from the dataset. +/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). +/// \return Shared pointer to the current Dataset +std::shared_ptr Coco(const std::string &dataset_dir, const std::string &annotation_file, + const std::string &task, const bool &decode, + const std::reference_wrapper sampler, const std::shared_ptr &cache = nullptr); class CSVDataset : public Dataset { @@ -774,7 +936,16 @@ std::shared_ptr CSV(const std::vector &dataset_files, c class ImageFolderDataset : public Dataset { public: explicit ImageFolderDataset(const std::string &dataset_dir, bool decode = false, - const std::shared_ptr &sampler = RandomSampler(), + const std::shared_ptr &sampler = std::make_shared(), + const std::set &extensions = {}, + const std::map &class_indexing = {}, + const std::shared_ptr &cache = nullptr); + explicit ImageFolderDataset(const std::string &dataset_dir, bool decode, Sampler *sampler, + const std::set &extensions = {}, + const std::map &class_indexing = {}, + const std::shared_ptr &cache = nullptr); + explicit ImageFolderDataset(const std::string &dataset_dir, bool decode, + const std::reference_wrapper sampler, const std::set &extensions = {}, const std::map &class_indexing = {}, const std::shared_ptr &cache = nullptr); @@ -787,14 +958,46 @@ class ImageFolderDataset : public Dataset { /// The generated dataset has two columns ["image", "label"] /// \param[in] dataset_dir Path to the root directory that contains the dataset /// \param[in] decode A flag to decode in ImageFolder -/// \param[in] sampler Object used to choose samples from the dataset. If sampler is not given, +/// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not +/// given, /// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()) /// \param[in] extensions File extensions to be read /// \param[in] class_indexing a class name to label map /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). /// \return Shared pointer to the current ImageFolderDataset -std::shared_ptr ImageFolder(const std::string &dataset_dir, bool decode = false, - const std::shared_ptr &sampler = RandomSampler(), +std::shared_ptr ImageFolder( + const std::string &dataset_dir, bool decode = false, + const std::shared_ptr &sampler = std::make_shared(), + const std::set &extensions = {}, const std::map &class_indexing = {}, + const std::shared_ptr &cache = nullptr); +/// \brief Function to create an ImageFolderDataset +/// \notes A source dataset that reads images from a tree of directories +/// All images within one folder have the same label +/// The generated dataset has two columns ["image", "label"] +/// \param[in] dataset_dir Path to the root directory that contains the dataset +/// \param[in] decode A flag to decode in ImageFolder +/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset. +/// \param[in] extensions File extensions to be read +/// \param[in] class_indexing a class name to label map +/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). +/// \return Shared pointer to the current ImageFolderDataset +std::shared_ptr ImageFolder(const std::string &dataset_dir, bool decode, Sampler *sampler, + const std::set &extensions = {}, + const std::map &class_indexing = {}, + const std::shared_ptr &cache = nullptr); +/// \brief Function to create an ImageFolderDataset +/// \notes A source dataset that reads images from a tree of directories +/// All images within one folder have the same label +/// The generated dataset has two columns ["image", "label"] +/// \param[in] dataset_dir Path to the root directory that contains the dataset +/// \param[in] decode A flag to decode in ImageFolder +/// \param[in] sampler Sampler object used to choose samples from the dataset. +/// \param[in] extensions File extensions to be read +/// \param[in] class_indexing a class name to label map +/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). +/// \return Shared pointer to the current ImageFolderDataset +std::shared_ptr ImageFolder(const std::string &dataset_dir, bool decode, + const std::reference_wrapper sampler, const std::set &extensions = {}, const std::map &class_indexing = {}, const std::shared_ptr &cache = nullptr); @@ -802,7 +1005,14 @@ std::shared_ptr ImageFolder(const std::string &dataset_dir, class ManifestDataset : public Dataset { public: explicit ManifestDataset(const std::string &dataset_file, const std::string &usage = "train", - const std::shared_ptr &sampler = RandomSampler(), + const std::shared_ptr &sampler = std::make_shared(), + const std::map &class_indexing = {}, bool decode = false, + const std::shared_ptr &cache = nullptr); + explicit ManifestDataset(const std::string &dataset_file, const std::string &usage, Sampler *sampler, + const std::map &class_indexing = {}, bool decode = false, + const std::shared_ptr &cache = nullptr); + explicit ManifestDataset(const std::string &dataset_file, const std::string &usage, + const std::reference_wrapper sampler, const std::map &class_indexing = {}, bool decode = false, const std::shared_ptr &cache = nullptr); ~ManifestDataset() = default; @@ -812,7 +1022,8 @@ class ManifestDataset : public Dataset { /// \notes The generated dataset has two columns ["image", "label"] /// \param[in] dataset_file The dataset file to be read /// \param[in] usage Need "train", "eval" or "inference" data (default="train") -/// \param[in] sampler Object used to choose samples from the dataset. If sampler is not given, +/// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not +/// given, /// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()) /// \param[in] class_indexing A str-to-int mapping from label name to index (default={}, the folder /// names will be sorted alphabetically and each class will be given a unique index starting from 0). @@ -820,19 +1031,56 @@ class ManifestDataset : public Dataset { /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). /// \return Shared pointer to the current ManifestDataset std::shared_ptr Manifest(const std::string &dataset_file, const std::string &usage = "train", - const std::shared_ptr &sampler = RandomSampler(), + const std::shared_ptr &sampler = std::make_shared(), + const std::map &class_indexing = {}, + bool decode = false, const std::shared_ptr &cache = nullptr); +/// \brief Function to create a ManifestDataset +/// \notes The generated dataset has two columns ["image", "label"] +/// \param[in] dataset_file The dataset file to be read +/// \param[in] usage Need "train", "eval" or "inference" data +/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset. +/// \param[in] class_indexing A str-to-int mapping from label name to index (default={}, the folder +/// names will be sorted alphabetically and each class will be given a unique index starting from 0). +/// \param[in] decode Decode the images after reading (default=false). +/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). +/// \return Shared pointer to the current ManifestDataset +std::shared_ptr Manifest(const std::string &dataset_file, const std::string &usage, Sampler *sampler, + const std::map &class_indexing = {}, + bool decode = false, const std::shared_ptr &cache = nullptr); +/// \brief Function to create a ManifestDataset +/// \notes The generated dataset has two columns ["image", "label"] +/// \param[in] dataset_file The dataset file to be read +/// \param[in] usage Need "train", "eval" or "inference" data +/// \param[in] sampler Sampler object used to choose samples from the dataset. +/// \param[in] class_indexing A str-to-int mapping from label name to index (default={}, the folder +/// names will be sorted alphabetically and each class will be given a unique index starting from 0). +/// \param[in] decode Decode the images after reading (default=false). +/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). +/// \return Shared pointer to the current ManifestDataset +std::shared_ptr Manifest(const std::string &dataset_file, const std::string &usage, + const std::reference_wrapper sampler, const std::map &class_indexing = {}, bool decode = false, const std::shared_ptr &cache = nullptr); class MindDataDataset : public Dataset { public: explicit MindDataDataset(const std::string &dataset_file, const std::vector &columns_list = {}, - const std::shared_ptr &sampler = RandomSampler(), + const std::shared_ptr &sampler = std::make_shared(), nlohmann::json padded_sample = nullptr, int64_t num_padded = 0); + explicit MindDataDataset(const std::string &dataset_file, const std::vector &columns_list, + Sampler *sampler, nlohmann::json padded_sample = nullptr, int64_t num_padded = 0); + explicit MindDataDataset(const std::string &dataset_file, const std::vector &columns_list, + const std::reference_wrapper sampler, nlohmann::json padded_sample = nullptr, + int64_t num_padded = 0); explicit MindDataDataset(const std::vector &dataset_files, const std::vector &columns_list = {}, - const std::shared_ptr &sampler = RandomSampler(), + const std::shared_ptr &sampler = std::make_shared(), nlohmann::json padded_sample = nullptr, int64_t num_padded = 0); + explicit MindDataDataset(const std::vector &dataset_files, const std::vector &columns_list, + Sampler *sampler, nlohmann::json padded_sample = nullptr, int64_t num_padded = 0); + explicit MindDataDataset(const std::vector &dataset_files, const std::vector &columns_list, + const std::reference_wrapper sampler, nlohmann::json padded_sample = nullptr, + int64_t num_padded = 0); ~MindDataDataset() = default; }; @@ -840,7 +1088,8 @@ class MindDataDataset : public Dataset { /// \param[in] dataset_file File name of one component of a mindrecord source. Other files with identical source /// in the same path will be found and loaded automatically. /// \param[in] columns_list List of columns to be read (default={}) -/// \param[in] sampler Object used to choose samples from the dataset. If sampler is not given, +/// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not +/// given, /// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()), /// supported sampler list: SubsetRandomSampler, PkSampler, RandomSampler, SequentialSampler, DistributedSampler. /// \param[in] padded_sample Samples will be appended to dataset, where keys are the same as column_list. @@ -848,13 +1097,38 @@ class MindDataDataset : public Dataset { /// \return Shared pointer to the current MindDataDataset std::shared_ptr MindData(const std::string &dataset_file, const std::vector &columns_list = {}, - const std::shared_ptr &sampler = RandomSampler(), + const std::shared_ptr &sampler = std::make_shared(), + nlohmann::json padded_sample = nullptr, int64_t num_padded = 0); +/// \brief Function to create a MindDataDataset +/// \param[in] dataset_file File name of one component of a mindrecord source. Other files with identical source +/// in the same path will be found and loaded automatically. +/// \param[in] columns_list List of columns to be read +/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset. +/// supported sampler list: SubsetRandomSampler, PkSampler, RandomSampler, SequentialSampler, DistributedSampler. +/// \param[in] padded_sample Samples will be appended to dataset, where keys are the same as column_list. +/// \param[in] num_padded Number of padding samples. Dataset size plus num_padded should be divisible by num_shards. +/// \return Shared pointer to the current MindDataDataset +std::shared_ptr MindData(const std::string &dataset_file, const std::vector &columns_list, + Sampler *sampler, nlohmann::json padded_sample = nullptr, + int64_t num_padded = 0); +/// \brief Function to create a MindDataDataset +/// \param[in] dataset_file File name of one component of a mindrecord source. Other files with identical source +/// in the same path will be found and loaded automatically. +/// \param[in] columns_list List of columns to be read +/// \param[in] sampler Sampler object used to choose samples from the dataset. +/// supported sampler list: SubsetRandomSampler, PkSampler, RandomSampler, SequentialSampler, DistributedSampler. +/// \param[in] padded_sample Samples will be appended to dataset, where keys are the same as column_list. +/// \param[in] num_padded Number of padding samples. Dataset size plus num_padded should be divisible by num_shards. +/// \return Shared pointer to the current MindDataDataset +std::shared_ptr MindData(const std::string &dataset_file, const std::vector &columns_list, + const std::reference_wrapper sampler, nlohmann::json padded_sample = nullptr, int64_t num_padded = 0); /// \brief Function to create a MindDataDataset /// \param[in] dataset_files List of dataset files to be read directly. /// \param[in] columns_list List of columns to be read (default={}) -/// \param[in] sampler Object used to choose samples from the dataset. If sampler is not given, +/// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not +/// given, /// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()), /// supported sampler list: SubsetRandomSampler, PkSampler, RandomSampler, SequentialSampler, DistributedSampler. /// \param[in] padded_sample Samples will be appended to dataset, where keys are the same as column_list. @@ -862,14 +1136,42 @@ std::shared_ptr MindData(const std::string &dataset_file, /// \return Shared pointer to the current MindDataDataset std::shared_ptr MindData(const std::vector &dataset_files, const std::vector &columns_list = {}, - const std::shared_ptr &sampler = RandomSampler(), + const std::shared_ptr &sampler = std::make_shared(), + nlohmann::json padded_sample = nullptr, int64_t num_padded = 0); +/// \brief Function to create a MindDataDataset +/// \param[in] dataset_files List of dataset files to be read directly. +/// \param[in] columns_list List of columns to be read +/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset. +/// supported sampler list: SubsetRandomSampler, PkSampler, RandomSampler, SequentialSampler, DistributedSampler. +/// \param[in] padded_sample Samples will be appended to dataset, where keys are the same as column_list. +/// \param[in] num_padded Number of padding samples. Dataset size plus num_padded should be divisible by num_shards. +/// \return Shared pointer to the current MindDataDataset +std::shared_ptr MindData(const std::vector &dataset_files, + const std::vector &columns_list, Sampler *sampler, + nlohmann::json padded_sample = nullptr, int64_t num_padded = 0); +/// \brief Function to create a MindDataDataset +/// \param[in] dataset_files List of dataset files to be read directly. +/// \param[in] columns_list List of columns to be read +/// \param[in] sampler Sampler object used to choose samples from the dataset. +/// supported sampler list: SubsetRandomSampler, PkSampler, RandomSampler, SequentialSampler, DistributedSampler. +/// \param[in] padded_sample Samples will be appended to dataset, where keys are the same as column_list. +/// \param[in] num_padded Number of padding samples. Dataset size plus num_padded should be divisible by num_shards. +/// \return Shared pointer to the current MindDataDataset +std::shared_ptr MindData(const std::vector &dataset_files, + const std::vector &columns_list, + const std::reference_wrapper sampler, nlohmann::json padded_sample = nullptr, int64_t num_padded = 0); #endif class MnistDataset : public Dataset { public: explicit MnistDataset(const std::string &dataset_dir, const std::string &usage = "all", - const std::shared_ptr &sampler = RandomSampler(), + const std::shared_ptr &sampler = std::make_shared(), + const std::shared_ptr &cache = nullptr); + explicit MnistDataset(const std::string &dataset_dir, const std::string &usage, Sampler *sampler, + const std::shared_ptr &cache = nullptr); + explicit MnistDataset(const std::string &dataset_dir, const std::string &usage, + const std::reference_wrapper sampler, const std::shared_ptr &cache = nullptr); ~MnistDataset() = default; }; @@ -878,12 +1180,32 @@ class MnistDataset : public Dataset { /// \notes The generated dataset has two columns ["image", "label"] /// \param[in] dataset_dir Path to the root directory that contains the dataset /// \param[in] usage of MNIST, can be "train", "test" or "all" (default = "all"). -/// \param[in] sampler Object used to choose samples from the dataset. If sampler is not given, +/// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not +/// given, /// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()) /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). /// \return Shared pointer to the current MnistDataset std::shared_ptr Mnist(const std::string &dataset_dir, const std::string &usage = "all", - const std::shared_ptr &sampler = RandomSampler(), + const std::shared_ptr &sampler = std::make_shared(), + const std::shared_ptr &cache = nullptr); +/// \brief Function to create a MnistDataset +/// \notes The generated dataset has two columns ["image", "label"] +/// \param[in] dataset_dir Path to the root directory that contains the dataset +/// \param[in] usage of MNIST, can be "train", "test" or "all" +/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset. +/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). +/// \return Shared pointer to the current MnistDataset +std::shared_ptr Mnist(const std::string &dataset_dir, const std::string &usage, Sampler *sampler, + const std::shared_ptr &cache = nullptr); +/// \brief Function to create a MnistDataset +/// \notes The generated dataset has two columns ["image", "label"] +/// \param[in] dataset_dir Path to the root directory that contains the dataset +/// \param[in] usage of MNIST, can be "train", "test" or "all" +/// \param[in] sampler Sampler object used to choose samples from the dataset. +/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). +/// \return Shared pointer to the current MnistDataset +std::shared_ptr Mnist(const std::string &dataset_dir, const std::string &usage, + const std::reference_wrapper sampler, const std::shared_ptr &cache = nullptr); #ifndef ENABLE_ANDROID /// \brief Function to create a ConcatDataset @@ -909,8 +1231,6 @@ class RandomDataDataset : public Dataset { /// \param[in] total_rows Number of rows for the dataset to generate (default=0, number of rows is random) /// \param[in] schema SchemaObj to set column type, data type and data shape /// \param[in] columns_list List of columns to be read (default={}, read all columns) -/// \param[in] sampler Object used to choose samples from the dataset. If sampler is not given, -/// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()) /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). /// \return Shared pointer to the current Dataset template > @@ -1027,7 +1347,14 @@ class VOCDataset : public Dataset { public: explicit VOCDataset(const std::string &dataset_dir, const std::string &task = "Segmentation", const std::string &usage = "train", const std::map &class_indexing = {}, - bool decode = false, const std::shared_ptr &sampler = RandomSampler(), + bool decode = false, const std::shared_ptr &sampler = std::make_shared(), + const std::shared_ptr &cache = nullptr); + explicit VOCDataset(const std::string &dataset_dir, const std::string &task, const std::string &usage, + const std::map &class_indexing, bool decode, Sampler *sampler, + const std::shared_ptr &cache = nullptr); + explicit VOCDataset(const std::string &dataset_dir, const std::string &task, const std::string &usage, + const std::map &class_indexing, bool decode, + const std::reference_wrapper sampler, const std::shared_ptr &cache = nullptr); ~VOCDataset() = default; }; @@ -1042,14 +1369,48 @@ class VOCDataset : public Dataset { /// \param[in] usage The type of data list text file to be read (default = "train"). /// \param[in] class_indexing A str-to-int mapping from label name to index, only valid in "Detection" task /// \param[in] decode Decode the images after reading -/// \param[in] sampler Object used to choose samples from the dataset. If sampler is not given, +/// \param[in] sampler Shared pointer to a sampler object used to choose samples from the dataset. If sampler is not +/// given, /// a `RandomSampler` will be used to randomly iterate the entire dataset (default = RandomSampler()) /// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). /// \return Shared pointer to the current Dataset std::shared_ptr VOC(const std::string &dataset_dir, const std::string &task = "Segmentation", const std::string &usage = "train", const std::map &class_indexing = {}, bool decode = false, - const std::shared_ptr &sampler = RandomSampler(), + const std::shared_ptr &sampler = std::make_shared(), + const std::shared_ptr &cache = nullptr); +/// \brief Function to create a VOCDataset +/// \notes The generated dataset has multi-columns : +/// - task='Detection', column: [['image', dtype=uint8], ['bbox', dtype=float32], ['label', dtype=uint32], +/// ['difficult', dtype=uint32], ['truncate', dtype=uint32]]. +/// - task='Segmentation', column: [['image', dtype=uint8], ['target',dtype=uint8]]. +/// \param[in] dataset_dir Path to the root directory that contains the dataset +/// \param[in] task Set the task type of reading voc data, now only support "Segmentation" or "Detection" +/// \param[in] usage The type of data list text file to be read. +/// \param[in] class_indexing A str-to-int mapping from label name to index, only valid in "Detection" task +/// \param[in] decode Decode the images after reading +/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset. +/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). +/// \return Shared pointer to the current Dataset +std::shared_ptr VOC(const std::string &dataset_dir, const std::string &task, const std::string &usage, + const std::map &class_indexing, bool decode, Sampler *sampler, + const std::shared_ptr &cache = nullptr); +/// \brief Function to create a VOCDataset +/// \notes The generated dataset has multi-columns : +/// - task='Detection', column: [['image', dtype=uint8], ['bbox', dtype=float32], ['label', dtype=uint32], +/// ['difficult', dtype=uint32], ['truncate', dtype=uint32]]. +/// - task='Segmentation', column: [['image', dtype=uint8], ['target',dtype=uint8]]. +/// \param[in] dataset_dir Path to the root directory that contains the dataset +/// \param[in] task Set the task type of reading voc data, now only support "Segmentation" or "Detection" +/// \param[in] usage The type of data list text file to be read. +/// \param[in] class_indexing A str-to-int mapping from label name to index, only valid in "Detection" task +/// \param[in] decode Decode the images after reading +/// \param[in] sampler Sampler object used to choose samples from the dataset. +/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used). +/// \return Shared pointer to the current Dataset +std::shared_ptr VOC(const std::string &dataset_dir, const std::string &task, const std::string &usage, + const std::map &class_indexing, bool decode, + const std::reference_wrapper sampler, const std::shared_ptr &cache = nullptr); /// \brief Function the create a cache to be attached to a dataset diff --git a/mindspore/ccsrc/minddata/dataset/include/samplers.h b/mindspore/ccsrc/minddata/dataset/include/samplers.h index 610010ec1b..21b1e7e253 100644 --- a/mindspore/ccsrc/minddata/dataset/include/samplers.h +++ b/mindspore/ccsrc/minddata/dataset/include/samplers.h @@ -26,76 +26,225 @@ namespace mindspore { namespace dataset { -class DistributedSamplerObj; -class PKSamplerObj; -class PreBuiltSamplerObj; -class RandomSamplerObj; -class SequentialSamplerObj; -class SubsetSamplerObj; -class SubsetRandomSamplerObj; -class WeightedRandomSamplerObj; - -/// Function to create a Distributed Sampler. -/// \notes A Sampler that access a shard of the dataset. -/// \param[in] num_shards - Number of shards to divide the dataset into. -/// \param[in] shard_id - Shard ID of the current shard within num_shards. -/// \param[in] shuffle - If true, the indices are shuffled. -/// \param[in] num_samples - The number of samples to draw (default to all elements). -/// \param[in] seed - The seed in use when shuffle is true. -/// \param[in] offset - The starting position where access to elements in the dataset begins. -/// \param[in] even_dist - If true, each shard would return the same number of rows (default to true). -/// If false the total rows returned by all the shards would not have overlap. -/// \return Shared pointer to the current Sampler. -std::shared_ptr DistributedSampler(int64_t num_shards, int64_t shard_id, bool shuffle = true, - int64_t num_samples = 0, uint32_t seed = 1, - int64_t offset = -1, bool even_dist = true); - -/// Function to create a PK Sampler. +// Abstract class to represent a sampler in the data pipeline. +/// \class Sampler samplers.h +/// \brief An abstract base class to represent a sampler in the data pipeline. +class Sampler : std::enable_shared_from_this { + friend class AlbumDataset; + friend class CelebADataset; + friend class Cifar10Dataset; + friend class Cifar100Dataset; + friend class CLUEDataset; + friend class CocoDataset; + friend class CSVDataset; + friend class ImageFolderDataset; + friend class ManifestDataset; + friend class MindDataDataset; + friend class MnistDataset; + friend class RandomDataDataset; + friend class TextFileDataset; + friend class TFRecordDataset; + friend class VOCDataset; + friend std::shared_ptr SelectSampler(int64_t, bool, int32_t, int32_t); + + public: + /// \brief Constructor + Sampler() {} + + /// \brief Destructor + ~Sampler() = default; + + /// \brief A virtual function to add a child sampler. + /// \param[in] child The child sampler to be added as a children of this sampler. + virtual void AddChild(std::shared_ptr child) { children_.push_back(child); } + + protected: + /// \brief Pure virtual function to convert a Sampler class into an IR Sampler object. + /// \return shared pointer to the newly created TensorOperation. + virtual std::shared_ptr Parse() = 0; + + std::vector> children_; +}; + +/// \brief A class to represent a Distributed Sampler in the data pipeline. +/// \notes A Sampler that accesses a shard of the dataset. +class DistributedSampler : public Sampler { + friend std::shared_ptr SelectSampler(int64_t, bool, int32_t, int32_t); + + public: + /// \brief Constructor + /// \param[in] num_shards - Number of shards to divide the dataset into. + /// \param[in] shard_id - Shard ID of the current shard within num_shards. + /// \param[in] shuffle - If true, the indices are shuffled. + /// \param[in] num_samples - The number of samples to draw (default to all elements). + /// \param[in] seed - The seed in use when shuffle is true. + /// \param[in] offset - The starting position where access to elements in the dataset begins. + /// \param[in] even_dist - If true, each shard would return the same number of rows (default to true). + /// If false the total rows returned by all the shards would not have overlap. + explicit DistributedSampler(int64_t num_shards, int64_t shard_id, bool shuffle = true, int64_t num_samples = 0, + uint32_t seed = 1, int64_t offset = -1, bool even_dist = true); + /// \brief Destructor. + ~DistributedSampler() = default; + + protected: + /// \brief Function to convert a Sampler into an IR SamplerObj. + /// \return shared pointer to the newly created SamplerObj. + std::shared_ptr Parse() override; + + private: + int64_t num_shards_; + int64_t shard_id_; + bool shuffle_; + int64_t num_samples_; + uint32_t seed_; + int64_t offset_; + bool even_dist_; +}; + +/// \brief A class to represent a PK Sampler in the data pipeline. /// \notes Samples K elements for each P class in the dataset. /// This will sample all classes. -/// \param[in] num_val - Number of elements to sample for each class. -/// \param[in] shuffle - If true, the class IDs are shuffled. -/// \param[in] num_samples - The number of samples to draw (default to all elements). -/// \return Shared pointer to the current Sampler. -std::shared_ptr PKSampler(int64_t num_val, bool shuffle = false, int64_t num_samples = 0); +class PKSampler : public Sampler { + friend std::shared_ptr SelectSampler(int64_t, bool, int32_t, int32_t); + + public: + /// \brief Constructor + /// \param[in] num_val - Number of elements to sample for each class. + /// \param[in] shuffle - If true, the class IDs are shuffled. + /// \param[in] num_samples - The number of samples to draw (default to all elements). + explicit PKSampler(int64_t num_val, bool shuffle = false, int64_t num_samples = 0); + + /// \brief Destructor. + ~PKSampler() = default; + + protected: + /// \brief Function to convert a Sampler into an IR SamplerObj. + /// \return shared pointer to the newly created SamplerObj. + std::shared_ptr Parse() override; -/// Function to create a Random Sampler. + private: + int64_t num_val_; + bool shuffle_; + int64_t num_samples_; +}; + +/// \brief A class to represent a Random Sampler in the data pipeline. /// \notes Samples the elements randomly. -/// \param[in] replacement - If true, put the sample ID back for the next draw. -/// \param[in] num_samples - The number of samples to draw (default to all elements). -/// \return Shared pointer to the current Sampler. -std::shared_ptr RandomSampler(bool replacement = false, int64_t num_samples = 0); +class RandomSampler : public Sampler { + friend std::shared_ptr SelectSampler(int64_t, bool, int32_t, int32_t); + + public: + /// \brief Constructor + /// \param[in] replacement - If true, put the sample ID back for the next draw. + /// \param[in] num_samples - The number of samples to draw (default to all elements). + explicit RandomSampler(bool replacement = false, int64_t num_samples = 0); + + /// \brief Destructor. + ~RandomSampler() = default; + + protected: + /// \brief Function to convert a Sampler into an IR SamplerObj. + /// \return shared pointer to the newly created SamplerObj. + std::shared_ptr Parse() override; -/// Function to create a Sequential Sampler. + private: + bool replacement_; + int64_t num_samples_; +}; + +/// \brief A class to represent a Sequential Sampler in the data pipeline. /// \notes Samples the dataset elements sequentially, same as not having a sampler. -/// \param[in] start_index - Index to start sampling at (default to start at first id). -/// \param[in] num_samples - The number of samples to draw (default to all elements). -/// \return Shared pointer to the current Sampler. -std::shared_ptr SequentialSampler(int64_t start_index = 0, int64_t num_samples = 0); +class SequentialSampler : public Sampler { + friend std::shared_ptr SelectSampler(int64_t, bool, int32_t, int32_t); + + public: + /// \brief Constructor + /// \param[in] start_index - Index to start sampling at (default to start at first id). + /// \param[in] num_samples - The number of samples to draw (default to all elements). + explicit SequentialSampler(int64_t start_index = 0, int64_t num_samples = 0); + + /// \brief Destructor. + ~SequentialSampler() = default; + + protected: + /// \brief Function to convert a Sampler into an IR SamplerObj. + /// \return shared pointer to the newly created SamplerObj. + std::shared_ptr Parse() override; -/// Function to create a Subset Sampler. + private: + int64_t start_index_; + int64_t num_samples_; +}; + +/// \brief A class to represent a Subset Sampler in the data pipeline. /// \notes Samples the elements from a sequence of indices. -/// \param[in] indices - A vector sequence of indices. -/// \param[in] num_samples - The number of samples to draw (default to all elements). -/// \return Shared pointer to the current Sampler. -std::shared_ptr SubsetSampler(std::vector indices, int64_t num_samples = 0); +class SubsetSampler : public Sampler { + friend std::shared_ptr SelectSampler(int64_t, bool, int32_t, int32_t); + + public: + /// \brief Constructor + /// \param[in] indices - A vector sequence of indices. + /// \param[in] num_samples - The number of samples to draw (default to all elements). + explicit SubsetSampler(std::vector indices, int64_t num_samples = 0); + + /// \brief Destructor. + ~SubsetSampler() = default; + + protected: + /// \brief Function to convert a Sampler into an IR SamplerObj. + /// \return shared pointer to the newly created SamplerObj. + std::shared_ptr Parse() override; -/// Function to create a Subset Random Sampler. + std::vector indices_; + int64_t num_samples_; +}; + +/// \brief A class to represent a Subset Random Sampler in the data pipeline. /// \notes Samples the elements randomly from a sequence of indices. -/// \param[in] indices - A vector sequence of indices. -/// \param[in] num_samples - The number of samples to draw (default to all elements). -/// \return Shared pointer to the current Sampler. -std::shared_ptr SubsetRandomSampler(std::vector indices, int64_t num_samples = 0); +class SubsetRandomSampler : public SubsetSampler { + friend std::shared_ptr SelectSampler(int64_t, bool, int32_t, int32_t); + + public: + /// \brief Constructor + /// \param[in] indices - A vector sequence of indices. + /// \param[in] num_samples - The number of samples to draw (default to all elements). + explicit SubsetRandomSampler(std::vector indices, int64_t num_samples = 0); + + /// \brief Destructor. + ~SubsetRandomSampler() = default; + + protected: + /// \brief Function to convert a Sampler into an IR SamplerObj. + /// \return shared pointer to the newly created SamplerObj. + std::shared_ptr Parse() override; +}; -/// Function to create a Weighted Random Sampler. +/// \brief A class to represent a Weighted Random Sampler in the data pipeline. /// \notes Samples the elements from [0, len(weights) - 1] randomly with the given /// weights (probabilities). -/// \param[in] weights - A vector sequence of weights, not necessarily summing up to 1. -/// \param[in] num_samples - The number of samples to draw (default to all elements). -/// \param[in] replacement - If true, put the sample ID back for the next draw. -/// \return Shared pointer to the current Sampler. -std::shared_ptr WeightedRandomSampler(std::vector weights, int64_t num_samples = 0, - bool replacement = true); +class WeightedRandomSampler : public Sampler { + friend std::shared_ptr SelectSampler(int64_t, bool, int32_t, int32_t); + + public: + /// \brief Constructor + /// \param[in] weights - A vector sequence of weights, not necessarily summing up to 1. + /// \param[in] num_samples - The number of samples to draw (default to all elements). + /// \param[in] replacement - If true, put the sample ID back for the next draw. + explicit WeightedRandomSampler(std::vector weights, int64_t num_samples = 0, bool replacement = true); + + /// \brief Destructor. + ~WeightedRandomSampler() = default; + + protected: + /// \brief Function to convert a Sampler into an IR SamplerObj. + /// \return shared pointer to the newly created SamplerObj. + std::shared_ptr Parse() override; + + private: + std::vector weights_; + int64_t num_samples_; + bool replacement_; +}; } // namespace dataset } // namespace mindspore diff --git a/tests/ut/cpp/dataset/c_api_cache_test.cc b/tests/ut/cpp/dataset/c_api_cache_test.cc index 8ff8ad97a1..95d0d42690 100644 --- a/tests/ut/cpp/dataset/c_api_cache_test.cc +++ b/tests/ut/cpp/dataset/c_api_cache_test.cc @@ -57,7 +57,7 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheCApiNestedCache) { // Create an ImageFolder Dataset, this folder_path only has 2 images in it std::string folder_path = datasets_root_path_ + "/testImageNetData/train/"; - std::shared_ptr ds = ImageFolder(folder_path, false, RandomSampler(), {}, {}, some_cache); + std::shared_ptr ds = ImageFolder(folder_path, false, std::make_shared(), {}, {}, some_cache); EXPECT_NE(ds, nullptr); // Create objects for the tensor ops @@ -85,7 +85,7 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheImageFolderCApi) { // Create an ImageFolder Dataset, this folder_path only has 2 images in it std::string folder_path = datasets_root_path_ + "/testImageNetData/train/"; - std::shared_ptr ds = ImageFolder(folder_path, false, RandomSampler(), {}, {}, some_cache); + std::shared_ptr ds = ImageFolder(folder_path, false, std::make_shared(), {}, {}, some_cache); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds @@ -128,7 +128,7 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheCocoCApi) { std::string folder_path = datasets_root_path_ + "/testCOCO/train/"; std::string annotation_file_path = datasets_root_path_ + "/testCOCO/annotations/train.json"; std::shared_ptr ds = - Coco(folder_path, annotation_file_path, "Detection", false, RandomSampler(), some_cache); + Coco(folder_path, annotation_file_path, "Detection", false, std::make_shared(), some_cache); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds @@ -169,7 +169,7 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheMnistCApi) { // Create a Mnist Dataset std::string folder_path = datasets_root_path_ + "/testMnistData/"; - std::shared_ptr ds = Mnist(folder_path, "all", RandomSampler(false, 10), some_cache); + std::shared_ptr ds = Mnist(folder_path, "all", std::make_shared(false, 10), some_cache); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds @@ -210,7 +210,7 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheCelebaCApi) { // Create a CelebA Dataset, this folder_path has 4 records in it std::string folder_path = datasets_root_path_ + "/testCelebAData/"; - std::shared_ptr ds = CelebA(folder_path, "all", RandomSampler(false, 10), false, {}, some_cache); + std::shared_ptr ds = CelebA(folder_path, "all", std::make_shared(false, 10), false, {}, some_cache); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds @@ -251,7 +251,7 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheManifestCApi) { // Create a Manifest Dataset, this file_path has 2 records in it std::string file_path = datasets_root_path_ + "/testManifestData/cpp.json"; - std::shared_ptr ds = Manifest(file_path, "train", RandomSampler(), {}, false, some_cache); + std::shared_ptr ds = Manifest(file_path, "train", std::make_shared(), {}, false, some_cache); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds @@ -292,7 +292,7 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheCifar10CApi) { // Create a Cifar10 Dataset std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds = Cifar10(folder_path, "all", RandomSampler(false, 10), some_cache); + std::shared_ptr ds = Cifar10(folder_path, "all", std::make_shared(false, 10), some_cache); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds @@ -333,7 +333,7 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheCifar100CApi) { // Create a Cifar100 Dataset std::string folder_path = datasets_root_path_ + "/testCifar100Data/"; - std::shared_ptr ds = Cifar100(folder_path, "all", RandomSampler(false, 10), some_cache); + std::shared_ptr ds = Cifar100(folder_path, "all", std::make_shared(false, 10), some_cache); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds @@ -374,7 +374,7 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheVocCApi) { // Create a VOC Dataset, this folder_path has 9 records in it std::string folder_path = datasets_root_path_ + "/testVOC2012/"; - std::shared_ptr ds = VOC(folder_path, "Detection", "train", {}, false, RandomSampler(), some_cache); + std::shared_ptr ds = VOC(folder_path, "Detection", "train", {}, false, std::make_shared(), some_cache); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds @@ -417,7 +417,7 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheAlbumCApi) { std::string schema_file = datasets_root_path_ + "/testAlbum/datasetSchema.json"; std::vector column_names = {"image", "label", "id"}; // Create a Album Dataset, 7 records in it - std::shared_ptr ds = Album(folder_path, schema_file, column_names, false, RandomSampler(), some_cache); + std::shared_ptr ds = Album(folder_path, schema_file, column_names, false, std::make_shared(), some_cache); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds @@ -770,9 +770,9 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCApiCacheShare1) { // Create an ImageFolder Dataset, this folder_path only has 2 images in it std::string folder_path = datasets_root_path_ + "/testImageNetData/train/"; - std::shared_ptr ds1 = ImageFolder(folder_path, false, RandomSampler(), {}, {}, some_cache); + std::shared_ptr ds1 = ImageFolder(folder_path, false, std::make_shared(), {}, {}, some_cache); EXPECT_NE(ds1, nullptr); - std::shared_ptr ds2 = ImageFolder(folder_path, false, RandomSampler(), {}, {}, some_cache); + std::shared_ptr ds2 = ImageFolder(folder_path, false, std::make_shared(), {}, {}, some_cache); EXPECT_NE(ds2, nullptr); // Create and launch the Execution Tree for ds1 @@ -824,9 +824,9 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCApiCacheShare2) { std::string folder_path = datasets_root_path_ + "/testImageNetData/train/"; // The first pipeline is ImageFolder with RandomSampler, the second pipeline is ImageFolder with SequentialSampler // Since sampler does not influence the data in the source, these two pipelines can share a common cache. - std::shared_ptr ds1 = ImageFolder(folder_path, true, RandomSampler(), {}, {}, some_cache); + std::shared_ptr ds1 = ImageFolder(folder_path, true, std::make_shared(), {}, {}, some_cache); EXPECT_NE(ds1, nullptr); - std::shared_ptr ds2 = ImageFolder(folder_path, true, SequentialSampler(), {}, {}, some_cache); + std::shared_ptr ds2 = ImageFolder(folder_path, true, std::make_shared(), {}, {}, some_cache); EXPECT_NE(ds2, nullptr); // Create and launch the Execution Tree for ds1 @@ -874,9 +874,9 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCApiCacheShareFailure1) { // Create an ImageFolder Dataset, this folder_path only has 2 images in it std::string folder_path = datasets_root_path_ + "/testImageNetData/train/"; - std::shared_ptr ds1 = ImageFolder(folder_path, true, RandomSampler(), {}, {}, some_cache); + std::shared_ptr ds1 = ImageFolder(folder_path, true, std::make_shared(), {}, {}, some_cache); EXPECT_NE(ds1, nullptr); - std::shared_ptr ds2 = ImageFolder(folder_path, false, RandomSampler(), {}, {}, some_cache); + std::shared_ptr ds2 = ImageFolder(folder_path, false, std::make_shared(), {}, {}, some_cache); EXPECT_NE(ds2, nullptr); // Create and launch the Execution Tree for ds1 diff --git a/tests/ut/cpp/dataset/c_api_dataset_album_test.cc b/tests/ut/cpp/dataset/c_api_dataset_album_test.cc index ce86fc0ec0..c1488b1e40 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_album_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_album_test.cc @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -174,7 +174,7 @@ TEST_F(MindDataTestPipeline, TestAlbumNumSamplers) { std::string schema_file = datasets_root_path_ + "/testAlbum/datasetSchema.json"; std::vector column_names = {"image", "label", "id"}; // Create a Album Dataset - std::shared_ptr ds = Album(folder_path, schema_file, column_names, true, SequentialSampler(0, 1)); + std::shared_ptr ds = Album(folder_path, schema_file, column_names, true, std::make_shared(0, 1)); EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset @@ -206,7 +206,7 @@ TEST_F(MindDataTestPipeline, TestAlbumError) { std::string schema_file = datasets_root_path_ + "/testAlbum/datasetSchema.json"; std::vector column_names = {"image", "label", "id"}; // Create an Album Dataset - std::shared_ptr ds = Album(folder_path, schema_file, column_names, true, SequentialSampler(0, 1)); + std::shared_ptr ds = Album(folder_path, schema_file, column_names, true, std::make_shared(0, 1)); EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset diff --git a/tests/ut/cpp/dataset/c_api_dataset_cifar_test.cc b/tests/ut/cpp/dataset/c_api_dataset_cifar_test.cc index 03ae37f4b7..e2fc4243ea 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_cifar_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_cifar_test.cc @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -30,7 +30,7 @@ TEST_F(MindDataTestPipeline, TestCifar10Dataset) { // Create a Cifar10 Dataset std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); + std::shared_ptr ds = Cifar10(folder_path, "all", std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset @@ -64,8 +64,8 @@ TEST_F(MindDataTestPipeline, TestCifar10DatasetWithPipeline) { // Create two Cifar10 Dataset std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds1 = Cifar10(folder_path, "all", RandomSampler(false, 10)); - std::shared_ptr ds2 = Cifar10(folder_path, "all", RandomSampler(false, 10)); + std::shared_ptr ds1 = Cifar10(folder_path, "all", std::make_shared(false, 10)); + std::shared_ptr ds2 = Cifar10(folder_path, "all", std::make_shared(false, 10)); EXPECT_NE(ds1, nullptr); EXPECT_NE(ds2, nullptr); @@ -168,7 +168,7 @@ TEST_F(MindDataTestPipeline, TestCifar100Dataset) { // Create a Cifar100 Dataset std::string folder_path = datasets_root_path_ + "/testCifar100Data/"; - std::shared_ptr ds = Cifar100(folder_path, "all", RandomSampler(false, 10)); + std::shared_ptr ds = Cifar100(folder_path, "all", std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset @@ -203,7 +203,7 @@ TEST_F(MindDataTestPipeline, TestCifar100Getters) { // Create a Cifar100 Dataset std::string folder_path = datasets_root_path_ + "/testCifar100Data/"; - std::shared_ptr ds = Cifar100(folder_path, "all", RandomSampler(false, 10)); + std::shared_ptr ds = Cifar100(folder_path, "all", std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); std::vector column_names = {"image", "coarse_label", "fine_label"}; @@ -230,7 +230,7 @@ TEST_F(MindDataTestPipeline, TestCifar100DatasetFail) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCifar100DatasetFail."; // Create a Cifar100 Dataset - std::shared_ptr ds = Cifar100("", "all", RandomSampler(false, 10)); + std::shared_ptr ds = Cifar100("", "all", std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset @@ -243,7 +243,7 @@ TEST_F(MindDataTestPipeline, TestCifar10DatasetFail) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCifar10DatasetFail."; // Create a Cifar10 Dataset - std::shared_ptr ds = Cifar10("", "all", RandomSampler(false, 10)); + std::shared_ptr ds = Cifar10("", "all", std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset @@ -299,7 +299,7 @@ TEST_F(MindDataTestPipeline, TestCifar100DatasetWithWrongSamplerFail) { // Create a Cifar100 Dataset std::string folder_path = datasets_root_path_ + "/testCifar100Data/"; - std::shared_ptr ds = Cifar100(folder_path, "all", RandomSampler(false, -10)); + std::shared_ptr ds = Cifar100(folder_path, "all", std::make_shared(false, -10)); EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset diff --git a/tests/ut/cpp/dataset/c_api_dataset_coco_test.cc b/tests/ut/cpp/dataset/c_api_dataset_coco_test.cc index 7073c5fa47..6f5475b47d 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_coco_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_coco_test.cc @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -138,7 +138,7 @@ TEST_F(MindDataTestPipeline, TestCocoDetection) { std::string folder_path = datasets_root_path_ + "/testCOCO/train"; std::string annotation_file = datasets_root_path_ + "/testCOCO/annotations/train.json"; - std::shared_ptr ds = Coco(folder_path, annotation_file, "Detection", false, SequentialSampler(0, 6)); + std::shared_ptr ds = Coco(folder_path, annotation_file, "Detection", false, std::make_shared(0, 6)); EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset @@ -220,7 +220,7 @@ TEST_F(MindDataTestPipeline, TestCocoKeypoint) { std::string folder_path = datasets_root_path_ + "/testCOCO/train"; std::string annotation_file = datasets_root_path_ + "/testCOCO/annotations/key_point.json"; - std::shared_ptr ds = Coco(folder_path, annotation_file, "Keypoint", false, SequentialSampler(0, 2)); + std::shared_ptr ds = Coco(folder_path, annotation_file, "Keypoint", false, std::make_shared(0, 2)); EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset @@ -275,7 +275,7 @@ TEST_F(MindDataTestPipeline, TestCocoPanoptic) { std::string folder_path = datasets_root_path_ + "/testCOCO/train"; std::string annotation_file = datasets_root_path_ + "/testCOCO/annotations/panoptic.json"; - std::shared_ptr ds = Coco(folder_path, annotation_file, "Panoptic", false, SequentialSampler(0, 2)); + std::shared_ptr ds = Coco(folder_path, annotation_file, "Panoptic", false, std::make_shared(0, 2)); EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset @@ -333,7 +333,7 @@ TEST_F(MindDataTestPipeline, TestCocoPanopticGetClassIndex) { std::string folder_path = datasets_root_path_ + "/testCOCO/train"; std::string annotation_file = datasets_root_path_ + "/testCOCO/annotations/panoptic.json"; - std::shared_ptr ds = Coco(folder_path, annotation_file, "Panoptic", false, SequentialSampler(0, 2)); + std::shared_ptr ds = Coco(folder_path, annotation_file, "Panoptic", false, std::make_shared(0, 2)); EXPECT_NE(ds, nullptr); std::vector>> class_index1 = ds->GetClassIndexing(); @@ -355,7 +355,7 @@ TEST_F(MindDataTestPipeline, TestCocoStuff) { std::string folder_path = datasets_root_path_ + "/testCOCO/train"; std::string annotation_file = datasets_root_path_ + "/testCOCO/annotations/train.json"; - std::shared_ptr ds = Coco(folder_path, annotation_file, "Stuff", false, SequentialSampler(0, 6)); + std::shared_ptr ds = Coco(folder_path, annotation_file, "Stuff", false, std::make_shared(0, 6)); EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset diff --git a/tests/ut/cpp/dataset/c_api_dataset_iterator_test.cc b/tests/ut/cpp/dataset/c_api_dataset_iterator_test.cc index cd8bd91d7c..5c69cb5005 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_iterator_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_iterator_test.cc @@ -1,6 +1,6 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -29,7 +29,7 @@ TEST_F(MindDataTestPipeline, TestIteratorEmptyColumn) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorEmptyColumn."; // Create a Cifar10 Dataset std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds = Cifar10(folder_path, "all", RandomSampler(false, 5)); + std::shared_ptr ds = Cifar10(folder_path, "all", std::make_shared(false, 5)); EXPECT_NE(ds, nullptr); // Create a Rename operation on ds @@ -65,7 +65,7 @@ TEST_F(MindDataTestPipeline, TestIteratorOneColumn) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorOneColumn."; // Create a Mnist Dataset std::string folder_path = datasets_root_path_ + "/testMnistData/"; - std::shared_ptr ds = Mnist(folder_path, "all", RandomSampler(false, 4)); + std::shared_ptr ds = Mnist(folder_path, "all", std::make_shared(false, 4)); EXPECT_NE(ds, nullptr); // Create a Batch operation on ds @@ -104,7 +104,7 @@ TEST_F(MindDataTestPipeline, TestIteratorReOrder) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorReOrder."; // Create a Cifar10 Dataset std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds = Cifar10(folder_path, "all", SequentialSampler(false, 4)); + std::shared_ptr ds = Cifar10(folder_path, "all", std::make_shared(false, 4)); EXPECT_NE(ds, nullptr); // Create a Take operation on ds @@ -144,7 +144,7 @@ TEST_F(MindDataTestPipeline, TestIteratorTwoColumns) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorTwoColumns."; // Create a VOC Dataset std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; - std::shared_ptr ds = VOC(folder_path, "Detection", "train", {}, false, SequentialSampler(0, 4)); + std::shared_ptr ds = VOC(folder_path, "Detection", "train", {}, false, std::make_shared(0, 4)); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds @@ -187,7 +187,7 @@ TEST_F(MindDataTestPipeline, TestIteratorWrongColumn) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestIteratorOneColumn."; // Create a Mnist Dataset std::string folder_path = datasets_root_path_ + "/testMnistData/"; - std::shared_ptr ds = Mnist(folder_path, "all", RandomSampler(false, 4)); + std::shared_ptr ds = Mnist(folder_path, "all", std::make_shared(false, 4)); EXPECT_NE(ds, nullptr); // Pass wrong column name diff --git a/tests/ut/cpp/dataset/c_api_dataset_manifest_test.cc b/tests/ut/cpp/dataset/c_api_dataset_manifest_test.cc index ee7ec08c6a..a63a729080 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_manifest_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_manifest_test.cc @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -147,7 +147,7 @@ TEST_F(MindDataTestPipeline, TestManifestDecode) { std::string file_path = datasets_root_path_ + "/testManifestData/cpp.json"; // Create a Manifest Dataset - std::shared_ptr ds = Manifest(file_path, "train", RandomSampler(), {}, true); + std::shared_ptr ds = Manifest(file_path, "train", std::make_shared(), {}, true); EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset @@ -218,7 +218,7 @@ TEST_F(MindDataTestPipeline, TestManifestClassIndex) { std::vector expected_label = {111, 222}; // Create a Manifest Dataset - std::shared_ptr ds = Manifest(file_path, "train", RandomSampler(), map, true); + std::shared_ptr ds = Manifest(file_path, "train", std::make_shared(), map, true); EXPECT_NE(ds, nullptr); std::vector>> class_index1 = ds->GetClassIndexing(); @@ -261,7 +261,7 @@ TEST_F(MindDataTestPipeline, TestManifestNumSamplers) { std::string file_path = datasets_root_path_ + "/testManifestData/cpp.json"; // Create a Manifest Dataset - std::shared_ptr ds = Manifest(file_path, "train", SequentialSampler(0, 1), {}, true); + std::shared_ptr ds = Manifest(file_path, "train", std::make_shared(0, 1), {}, true); EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset diff --git a/tests/ut/cpp/dataset/c_api_dataset_minddata_test.cc b/tests/ut/cpp/dataset/c_api_dataset_minddata_test.cc index fa878a0017..d58301764a 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_minddata_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_minddata_test.cc @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -182,7 +182,7 @@ TEST_F(MindDataTestPipeline, TestMindDataSuccess5) { // Pass one mindrecord shard file to parse dataset info, and search for other mindrecord files with same dataset info, // thus all records in imagenet.mindrecord0 ~ imagenet.mindrecord3 will be read std::string file_path1 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0"; - std::shared_ptr ds = MindData(file_path1, {}, SequentialSampler(0, 3)); + std::shared_ptr ds = MindData(file_path1, {}, std::make_shared(0, 3)); EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset @@ -223,26 +223,26 @@ TEST_F(MindDataTestPipeline, TestMindDataSuccess6) { std::vector file_list = {file_path1}; // Check sequential sampler, output number is 5 - std::shared_ptr ds1 = MindData(file_list, {}, SequentialSampler(0, 10)); + std::shared_ptr ds1 = MindData(file_list, {}, std::make_shared(0, 10)); EXPECT_NE(ds1, nullptr); // Check random sampler, output number is 5, same rows with file - std::shared_ptr ds2 = MindData(file_list, {}, RandomSampler(false, 10)); + std::shared_ptr ds2 = MindData(file_list, {}, std::make_shared(false, 10)); EXPECT_NE(ds2, nullptr); // Check pk sampler, output number is 2, get 2 samples with label 0 - std::shared_ptr ds3 = MindData(file_list, {}, PKSampler(2, false, 10)); + std::shared_ptr ds3 = MindData(file_list, {}, std::make_shared(2, false, 10)); EXPECT_NE(ds3, nullptr); // Check distributed sampler, output number is 3, get 3 samples in shard 0 - std::shared_ptr ds4 = MindData(file_list, {}, DistributedSampler(2, 0, false, 10)); + std::shared_ptr ds4 = MindData(file_list, {}, std::make_shared(2, 0, false, 10)); EXPECT_NE(ds4, nullptr); // Check distributed sampler get 3 samples with indice 0, 1 ,2 - std::shared_ptr ds5 = MindData(file_list, {}, SubsetRandomSampler({0, 1, 2}, 10)); + std::shared_ptr ds5 = MindData(file_list, {}, new SubsetRandomSampler({0, 1, 2}, 10)); EXPECT_NE(ds5, nullptr); - std::shared_ptr ds6 = MindData(file_list, {}, SubsetSampler({1, 2}, 10)); + std::shared_ptr ds6 = MindData(file_list, {}, new SubsetSampler({1, 2}, 10)); EXPECT_NE(ds5, nullptr); std::vector> ds = {ds1, ds2, ds3, ds4, ds5, ds6}; @@ -283,7 +283,7 @@ TEST_F(MindDataTestPipeline, TestMindDataSuccess7) { // Pass a list of mindrecord file name, files in list will be read directly but not search for related files std::string file_path1 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0"; std::vector file_list = {file_path1}; - std::shared_ptr ds = MindData(file_list, {"file_name", "label"}, SequentialSampler(), pad, 4); + std::shared_ptr ds = MindData(file_list, {"file_name", "label"}, std::make_shared(), pad, 4); EXPECT_NE(ds, nullptr); // Create a Skip operation on ds, skip original data in mindrecord and get padded samples @@ -332,7 +332,7 @@ TEST_F(MindDataTestPipeline, TestMindDataSuccess8) { // Pass a list of mindrecord file name, files in list will be read directly but not search for related files std::string file_path1 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0"; std::vector file_list = {file_path1}; - std::shared_ptr ds = MindData(file_list, {"file_name", "label"}, SequentialSampler(), pad, 4); + std::shared_ptr ds = MindData(file_list, {"file_name", "label"}, std::make_shared(), pad, 4); EXPECT_NE(ds, nullptr); std::vector types = ds->GetOutputTypes(); @@ -400,12 +400,12 @@ TEST_F(MindDataTestPipeline, TestMindDataSuccess9) { // Pass a list of mindrecord file name, files in list will be read directly but not search for related files std::string file_path1 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0"; std::vector file_list = {file_path1}; - std::shared_ptr ds1 = MindData(file_list, {"file_name", "label"}, SequentialSampler(), pad, 4); + std::shared_ptr ds1 = MindData(file_list, {"file_name", "label"}, std::make_shared(), pad, 4); EXPECT_NE(ds1, nullptr); ds1 = ds1->Skip(5); EXPECT_NE(ds1, nullptr); - std::shared_ptr ds2 = MindData(file_list, {"file_name", "label"}, SequentialSampler(), pad, 4); + std::shared_ptr ds2 = MindData(file_list, {"file_name", "label"}, std::make_shared(), pad, 4); EXPECT_NE(ds2, nullptr); ds2 = ds2->Skip(5); EXPECT_NE(ds2, nullptr); @@ -534,7 +534,7 @@ TEST_F(MindDataTestPipeline, TestMindDataFail3) { // Create a MindData Dataset with unsupported sampler std::string file_path1 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0"; - std::shared_ptr ds1 = MindData(file_path1, {}, WeightedRandomSampler({1, 1, 1, 1})); + std::shared_ptr ds1 = MindData(file_path1, {}, new WeightedRandomSampler({1, 1, 1, 1})); EXPECT_NE(ds1, nullptr); // Create an iterator over the result of the above dataset @@ -558,7 +558,7 @@ TEST_F(MindDataTestPipeline, TestMindDataFail4) { // Create a MindData Dataset std::string file_path1 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0"; - std::shared_ptr ds1 = MindData(file_path1, {}, RandomSampler(), nullptr, 2); + std::shared_ptr ds1 = MindData(file_path1, {}, std::make_shared(), nullptr, 2); EXPECT_NE(ds1, nullptr); // Create an iterator over the result of the above dataset @@ -573,7 +573,7 @@ TEST_F(MindDataTestPipeline, TestMindDataFail4) { // Create a MindData Dataset std::string file_path2 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0"; - std::shared_ptr ds2 = MindData(file_path2, {"label"}, RandomSampler(), pad, -2); + std::shared_ptr ds2 = MindData(file_path2, {"label"}, std::make_shared(), pad, -2); EXPECT_NE(ds2, nullptr); // Create an iterator over the result of the above dataset @@ -583,7 +583,7 @@ TEST_F(MindDataTestPipeline, TestMindDataFail4) { // Create a MindData Dataset std::string file_path3 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0"; - std::shared_ptr ds3 = MindData(file_path3, {}, RandomSampler(), pad, 1); + std::shared_ptr ds3 = MindData(file_path3, {}, std::make_shared(), pad, 1); EXPECT_NE(ds3, nullptr); // Create an iterator over the result of the above dataset @@ -598,7 +598,7 @@ TEST_F(MindDataTestPipeline, TestMindDataFail4) { // Create a MindData Dataset std::string file_path4 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0"; - std::shared_ptr ds4 = MindData(file_path4, {"file_name", "label"}, RandomSampler(), pad2, 1); + std::shared_ptr ds4 = MindData(file_path4, {"file_name", "label"}, std::make_shared(), pad2, 1); EXPECT_NE(ds4, nullptr); // Create an iterator over the result of the above dataset diff --git a/tests/ut/cpp/dataset/c_api_dataset_ops_test.cc b/tests/ut/cpp/dataset/c_api_dataset_ops_test.cc index 546628f8f7..5741ba16a0 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_ops_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_ops_test.cc @@ -76,7 +76,7 @@ TEST_F(MindDataTestPipeline, TestBatchAndRepeat) { // Create a Mnist Dataset std::string folder_path = datasets_root_path_ + "/testMnistData/"; - std::shared_ptr ds = Mnist(folder_path, "all", RandomSampler(false, 10)); + std::shared_ptr ds = Mnist(folder_path, "all", std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds @@ -118,7 +118,7 @@ TEST_F(MindDataTestPipeline, TestBucketBatchByLengthSuccess1) { // Create a Mnist Dataset std::string folder_path = datasets_root_path_ + "/testMnistData/"; - std::shared_ptr ds = Mnist(folder_path, "all", RandomSampler(false, 10)); + std::shared_ptr ds = Mnist(folder_path, "all", std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a BucketBatchByLength operation on ds @@ -154,7 +154,7 @@ TEST_F(MindDataTestPipeline, TestBucketBatchByLengthSuccess2) { // Create a Mnist Dataset std::string folder_path = datasets_root_path_ + "/testMnistData/"; - std::shared_ptr ds = Mnist(folder_path, "all", RandomSampler(false, 10)); + std::shared_ptr ds = Mnist(folder_path, "all", std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a BucketBatchByLength operation on ds @@ -192,7 +192,7 @@ TEST_F(MindDataTestPipeline, TestBucketBatchByLengthFail1) { // Create a Mnist Dataset std::string folder_path = datasets_root_path_ + "/testMnistData/"; - std::shared_ptr ds = Mnist(folder_path, "all", RandomSampler(false, 10)); + std::shared_ptr ds = Mnist(folder_path, "all", std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a BucketBatchByLength operation on ds @@ -212,7 +212,7 @@ TEST_F(MindDataTestPipeline, TestBucketBatchByLengthFail2) { // Create a Mnist Dataset std::string folder_path = datasets_root_path_ + "/testMnistData/"; - std::shared_ptr ds = Mnist(folder_path, "all", RandomSampler(false, 10)); + std::shared_ptr ds = Mnist(folder_path, "all", std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a BucketBatchByLength operation on ds @@ -232,7 +232,7 @@ TEST_F(MindDataTestPipeline, TestBucketBatchByLengthFail3) { // Create a Mnist Dataset std::string folder_path = datasets_root_path_ + "/testMnistData/"; - std::shared_ptr ds = Mnist(folder_path, "all", RandomSampler(false, 10)); + std::shared_ptr ds = Mnist(folder_path, "all", std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a BucketBatchByLength operation on ds @@ -252,7 +252,7 @@ TEST_F(MindDataTestPipeline, TestBucketBatchByLengthFail4) { // Create a Mnist Dataset std::string folder_path = datasets_root_path_ + "/testMnistData/"; - std::shared_ptr ds = Mnist(folder_path, "all", RandomSampler(false, 10)); + std::shared_ptr ds = Mnist(folder_path, "all", std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a BucketBatchByLength operation on ds @@ -272,7 +272,7 @@ TEST_F(MindDataTestPipeline, TestBucketBatchByLengthFail5) { // Create a Mnist Dataset std::string folder_path = datasets_root_path_ + "/testMnistData/"; - std::shared_ptr ds = Mnist(folder_path, "all", RandomSampler(false, 10)); + std::shared_ptr ds = Mnist(folder_path, "all", std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a BucketBatchByLength operation on ds @@ -292,7 +292,7 @@ TEST_F(MindDataTestPipeline, TestBucketBatchByLengthFail6) { // Create a Mnist Dataset std::string folder_path = datasets_root_path_ + "/testMnistData/"; - std::shared_ptr ds = Mnist(folder_path, "all", RandomSampler(false, 10)); + std::shared_ptr ds = Mnist(folder_path, "all", std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a BucketBatchByLength operation on ds ds = ds->BucketBatchByLength({"image"}, {1, 2}, {1, -2, 3}); @@ -311,7 +311,7 @@ TEST_F(MindDataTestPipeline, TestBucketBatchByLengthFail7) { // Create a Mnist Dataset std::string folder_path = datasets_root_path_ + "/testMnistData/"; - std::shared_ptr ds = Mnist(folder_path, "all", RandomSampler(false, 10)); + std::shared_ptr ds = Mnist(folder_path, "all", std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a BucketBatchByLength operation on ds @@ -331,9 +331,9 @@ TEST_F(MindDataTestPipeline, TestConcatFail1) { // Create an ImageFolder Dataset // Column names: {"image", "label"} std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); - std::shared_ptr ds2 = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds2 = ImageFolder(folder_path, true, std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a Rename operation on ds @@ -362,7 +362,7 @@ TEST_F(MindDataTestPipeline, TestConcatFail2) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a Concat operation on the ds @@ -382,7 +382,7 @@ TEST_F(MindDataTestPipeline, TestConcatFail3) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a Concat operation on the ds @@ -402,7 +402,7 @@ TEST_F(MindDataTestPipeline, TestConcatFail4) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a Concat operation on the ds @@ -422,10 +422,10 @@ TEST_F(MindDataTestPipeline, TestConcatFail5) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds1 = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds1 = ImageFolder(folder_path, true, std::make_shared(false, 10)); EXPECT_NE(ds1, nullptr); - std::shared_ptr ds2 = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds2 = ImageFolder(folder_path, true, std::make_shared(false, 10)); EXPECT_NE(ds2, nullptr); // Create a Project operation on ds @@ -451,13 +451,13 @@ TEST_F(MindDataTestPipeline, TestConcatSuccess) { // Create an ImageFolder Dataset // Column names: {"image", "label"} std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a Cifar10 Dataset // Column names: {"image", "label"} folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds2 = Cifar10(folder_path, "all", RandomSampler(false, 9)); + std::shared_ptr ds2 = Cifar10(folder_path, "all", std::make_shared(false, 9)); EXPECT_NE(ds2, nullptr); // Create a Project operation on ds @@ -502,13 +502,13 @@ TEST_F(MindDataTestPipeline, TestConcatGetDatasetSize) { // Create an ImageFolder Dataset // Column names: {"image", "label"} std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a Cifar10 Dataset // Column names: {"image", "label"} folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds2 = Cifar10(folder_path, "all", RandomSampler(false, 9)); + std::shared_ptr ds2 = Cifar10(folder_path, "all", std::make_shared(false, 9)); EXPECT_NE(ds2, nullptr); // Create a Project operation on ds @@ -531,13 +531,13 @@ TEST_F(MindDataTestPipeline, TestConcatSuccess2) { // Create an ImageFolder Dataset // Column names: {"image", "label"} std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a Cifar10 Dataset // Column names: {"image", "label"} folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds2 = Cifar10(folder_path, "all", RandomSampler(false, 9)); + std::shared_ptr ds2 = Cifar10(folder_path, "all", std::make_shared(false, 9)); EXPECT_NE(ds2, nullptr); // Create a Project operation on ds @@ -755,7 +755,7 @@ TEST_F(MindDataTestPipeline, TestImageFolderBatchAndRepeat) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds @@ -796,7 +796,7 @@ TEST_F(MindDataTestPipeline, TestPipelineGetDatasetSize) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds @@ -818,7 +818,7 @@ TEST_F(MindDataTestPipeline, TestDistributedGetDatasetSize1) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, DistributedSampler(4, 0, false, 10)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(4, 0, false, 10)); EXPECT_NE(ds, nullptr); // num_per_shard is equal to 44/4 = 11 which is more than num_samples = 10, so the output is 10 @@ -849,7 +849,7 @@ TEST_F(MindDataTestPipeline, TestDistributedGetDatasetSize2) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, DistributedSampler(4, 0, false, 15)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(4, 0, false, 15)); EXPECT_NE(ds, nullptr); // num_per_shard is equal to 44/4 = 11 which is less than num_samples = 15, so the output is 11 @@ -879,7 +879,7 @@ TEST_F(MindDataTestPipeline, TestProjectMap) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds @@ -933,7 +933,7 @@ TEST_F(MindDataTestPipeline, TestProjectDuplicateColumnFail) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 3)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 3)); EXPECT_NE(ds, nullptr); // Create objects for the tensor ops @@ -962,7 +962,7 @@ TEST_F(MindDataTestPipeline, TestMapDuplicateColumnFail) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create objects for the tensor ops @@ -1002,7 +1002,7 @@ TEST_F(MindDataTestPipeline, TestProjectMapAutoInjection) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds @@ -1054,7 +1054,7 @@ TEST_F(MindDataTestPipeline, TestRenameFail1) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds @@ -1078,7 +1078,7 @@ TEST_F(MindDataTestPipeline, TestRenameFail2) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a Rename operation on ds @@ -1097,7 +1097,7 @@ TEST_F(MindDataTestPipeline, TestRenameFail3) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a Rename operation on ds @@ -1124,7 +1124,7 @@ TEST_F(MindDataTestPipeline, TestRenameSuccess) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds @@ -1174,7 +1174,7 @@ TEST_F(MindDataTestPipeline, TestRepeatDefault) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds @@ -1217,7 +1217,7 @@ TEST_F(MindDataTestPipeline, TestRepeatOne) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds @@ -1256,7 +1256,7 @@ TEST_F(MindDataTestPipeline, TestRepeatFail1) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds @@ -1276,7 +1276,7 @@ TEST_F(MindDataTestPipeline, TestRepeatFail2) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds @@ -1295,7 +1295,7 @@ TEST_F(MindDataTestPipeline, TestShuffleDataset) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a Shuffle operation on ds @@ -1341,7 +1341,7 @@ TEST_F(MindDataTestPipeline, TestSkipDataset) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a Skip operation on ds @@ -1379,7 +1379,7 @@ TEST_F(MindDataTestPipeline, TestSkipTakeRepeat) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 6)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 6)); // Create a Skip operation on ds int32_t count = 0; @@ -1424,7 +1424,7 @@ TEST_F(MindDataTestPipeline, TestSkipGetDatasetSize) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a Skip operation on ds @@ -1440,7 +1440,7 @@ TEST_F(MindDataTestPipeline, TestSkipDatasetError1) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a Skip operation on ds with invalid count input @@ -1459,7 +1459,7 @@ TEST_F(MindDataTestPipeline, TestTakeDatasetDefault) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 7)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 7)); EXPECT_NE(ds, nullptr); // Create a Take operation on ds, default count = -1 @@ -1496,7 +1496,7 @@ TEST_F(MindDataTestPipeline, TestTakeGetDatasetSize) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 7)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 7)); EXPECT_NE(ds, nullptr); // Create a Take operation on ds, default count = -1 @@ -1511,7 +1511,7 @@ TEST_F(MindDataTestPipeline, TestTakeDatasetError1) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a Take operation on ds with invalid count input @@ -1540,7 +1540,7 @@ TEST_F(MindDataTestPipeline, TestTakeDatasetNormal) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 8)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 8)); EXPECT_NE(ds, nullptr); // Create a Take operation on ds @@ -1577,7 +1577,7 @@ TEST_F(MindDataTestPipeline, TestTensorOpsAndMap) { // Create a Mnist Dataset std::string folder_path = datasets_root_path_ + "/testMnistData/"; - std::shared_ptr ds = Mnist(folder_path, "all", RandomSampler(false, 20)); + std::shared_ptr ds = Mnist(folder_path, "all", std::make_shared(false, 20)); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds @@ -1631,11 +1631,11 @@ TEST_F(MindDataTestPipeline, TestZipFail) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create an ImageFolder Dataset - std::shared_ptr ds1 = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds1 = ImageFolder(folder_path, true, std::make_shared(false, 10)); EXPECT_NE(ds1, nullptr); // Create a Zip operation on the datasets @@ -1659,7 +1659,7 @@ TEST_F(MindDataTestPipeline, TestZipFail2) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a Zip operation on the datasets @@ -1679,7 +1679,7 @@ TEST_F(MindDataTestPipeline, TestZipSuccess) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a Project operation on ds @@ -1688,7 +1688,7 @@ TEST_F(MindDataTestPipeline, TestZipSuccess) { EXPECT_NE(ds, nullptr); // Create an ImageFolder Dataset - std::shared_ptr ds1 = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds1 = ImageFolder(folder_path, true, std::make_shared(false, 10)); EXPECT_NE(ds1, nullptr); // Create a Rename operation on ds (so that the 3 datasets we are going to zip have distinct column names) @@ -1696,7 +1696,7 @@ TEST_F(MindDataTestPipeline, TestZipSuccess) { EXPECT_NE(ds1, nullptr); folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds2 = Cifar10(folder_path, "all", RandomSampler(false, 10)); + std::shared_ptr ds2 = Cifar10(folder_path, "all", std::make_shared(false, 10)); EXPECT_NE(ds2, nullptr); // Create a Project operation on ds @@ -1749,7 +1749,7 @@ TEST_F(MindDataTestPipeline, TestZipGetDatasetSize) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 2)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 2)); EXPECT_NE(ds, nullptr); // Create a Project operation on ds @@ -1758,7 +1758,7 @@ TEST_F(MindDataTestPipeline, TestZipGetDatasetSize) { EXPECT_NE(ds, nullptr); // Create an ImageFolder Dataset - std::shared_ptr ds1 = ImageFolder(folder_path, true, RandomSampler(false, 3)); + std::shared_ptr ds1 = ImageFolder(folder_path, true, std::make_shared(false, 3)); EXPECT_NE(ds1, nullptr); // Create a Rename operation on ds (so that the 3 datasets we are going to zip have distinct column names) @@ -1766,7 +1766,7 @@ TEST_F(MindDataTestPipeline, TestZipGetDatasetSize) { EXPECT_NE(ds1, nullptr); folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds2 = Cifar10(folder_path, "all", RandomSampler(false, 5)); + std::shared_ptr ds2 = Cifar10(folder_path, "all", std::make_shared(false, 5)); EXPECT_NE(ds2, nullptr); // Create a Project operation on ds @@ -1787,9 +1787,9 @@ TEST_F(MindDataTestPipeline, TestZipSuccess2) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 9)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 9)); EXPECT_NE(ds, nullptr); - std::shared_ptr ds2 = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds2 = ImageFolder(folder_path, true, std::make_shared(false, 10)); EXPECT_NE(ds2, nullptr); // Create a Rename operation on ds (so that the 2 datasets we are going to zip have distinct column names) @@ -1840,7 +1840,7 @@ TEST_F(MindDataTestPipeline, TestNumWorkersValidate) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, false, SequentialSampler(0, 1)); + std::shared_ptr ds = ImageFolder(folder_path, false, std::make_shared(0, 1)); // ds needs to be non nullptr otherwise, the subsequent logic will core dump ASSERT_NE(ds, nullptr); diff --git a/tests/ut/cpp/dataset/c_api_dataset_save.cc b/tests/ut/cpp/dataset/c_api_dataset_save.cc index f23d4c957d..13807a27f9 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_save.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_save.cc @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -30,7 +30,7 @@ TEST_F(MindDataTestPipeline, TestSaveCifar10AndLoad) { // Stage 1: load original dataset // Create a Cifar10 Dataset std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds = Cifar10(folder_path, "all", SequentialSampler(0, 10)); + std::shared_ptr ds = Cifar10(folder_path, "all", std::make_shared(0, 10)); EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset @@ -70,7 +70,7 @@ TEST_F(MindDataTestPipeline, TestSaveCifar10AndLoad) { // Stage 3: Load dataset from file output by stage 2 // Create a MindData Dataset - std::shared_ptr ds_minddata = MindData(temp_file, {}, SequentialSampler(0, 10)); + std::shared_ptr ds_minddata = MindData(temp_file, {}, std::make_shared(0, 10)); // Create objects for the tensor ops // uint32 will be casted to int64 implicitly in mindrecord file, so we have to cast it back to uint32 @@ -119,7 +119,7 @@ TEST_F(MindDataTestPipeline, TestSaveFail) { // Create a Cifar10 Dataset std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds = Cifar10(folder_path, "all", SequentialSampler(0, 10)); + std::shared_ptr ds = Cifar10(folder_path, "all", std::make_shared(0, 10)); EXPECT_NE(ds, nullptr); // fail with invalid dataset_path diff --git a/tests/ut/cpp/dataset/c_api_dataset_voc_test.cc b/tests/ut/cpp/dataset/c_api_dataset_voc_test.cc index 4607472bd5..cd24bd650f 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_voc_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_voc_test.cc @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -35,7 +35,7 @@ TEST_F(MindDataTestPipeline, TestVOCClassIndex) { class_index["cat"] = 1; class_index["train"] = 9; - std::shared_ptr ds = VOC(folder_path, "Detection", "train", class_index, false, SequentialSampler(0, 6)); + std::shared_ptr ds = VOC(folder_path, "Detection", "train", class_index, false, std::make_shared(0, 6)); EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset @@ -81,7 +81,7 @@ TEST_F(MindDataTestPipeline, TestVOCGetClassIndex) { class_index["cat"] = 1; class_index["train"] = 9; - std::shared_ptr ds = VOC(folder_path, "Detection", "train", class_index, false, SequentialSampler(0, 6)); + std::shared_ptr ds = VOC(folder_path, "Detection", "train", class_index, false, std::make_shared(0, 6)); EXPECT_NE(ds, nullptr); std::vector>> class_index1 = ds->GetClassIndexing(); @@ -104,7 +104,7 @@ TEST_F(MindDataTestPipeline, TestVOCGetters) { class_index["cat"] = 1; class_index["train"] = 9; - std::shared_ptr ds = VOC(folder_path, "Detection", "train", class_index, false, SequentialSampler(0, 6)); + std::shared_ptr ds = VOC(folder_path, "Detection", "train", class_index, false, std::make_shared(0, 6)); EXPECT_NE(ds, nullptr); ds = ds->Batch(2); @@ -120,7 +120,7 @@ TEST_F(MindDataTestPipeline, TestVOCDetection) { // Create a VOC Dataset std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; - std::shared_ptr ds = VOC(folder_path, "Detection", "train", {}, false, SequentialSampler(0, 4)); + std::shared_ptr ds = VOC(folder_path, "Detection", "train", {}, false, std::make_shared(0, 4)); EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset @@ -166,7 +166,7 @@ TEST_F(MindDataTestPipeline, TestVOCInvalidTaskOrModeError1) { // Create a VOC Dataset std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; - std::shared_ptr ds1 = VOC(folder_path, "Classification", "train", {}, false, SequentialSampler(0, 3)); + std::shared_ptr ds1 = VOC(folder_path, "Classification", "train", {}, false, std::make_shared(0, 3)); EXPECT_NE(ds1, nullptr); // Create an iterator over the result of the above dataset @@ -174,7 +174,7 @@ TEST_F(MindDataTestPipeline, TestVOCInvalidTaskOrModeError1) { // Expect failure: invalid Manifest input, invalid task EXPECT_EQ(iter1, nullptr); - std::shared_ptr ds2 = VOC(folder_path, "Segmentation", "validation", {}, false, RandomSampler(false, 4)); + std::shared_ptr ds2 = VOC(folder_path, "Segmentation", "validation", {}, false, std::make_shared(false, 4)); EXPECT_NE(ds2, nullptr); // Create an iterator over the result of the above dataset @@ -188,7 +188,7 @@ TEST_F(MindDataTestPipeline, TestVOCSegmentation) { // Create a VOC Dataset std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; - std::shared_ptr ds = VOC(folder_path, "Segmentation", "train", {}, false, SequentialSampler(0, 3)); + std::shared_ptr ds = VOC(folder_path, "Segmentation", "train", {}, false, std::make_shared(0, 3)); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds @@ -240,7 +240,7 @@ TEST_F(MindDataTestPipeline, TestVOCSegmentationError2) { std::map class_index; class_index["car"] = 0; std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; - std::shared_ptr ds = VOC(folder_path, "Segmentation", "train", class_index, false, RandomSampler(false, 6)); + std::shared_ptr ds = VOC(folder_path, "Segmentation", "train", class_index, false, std::make_shared(false, 6)); EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset diff --git a/tests/ut/cpp/dataset/c_api_datasets_test.cc b/tests/ut/cpp/dataset/c_api_datasets_test.cc index c69f70f56d..1bb1374bb2 100644 --- a/tests/ut/cpp/dataset/c_api_datasets_test.cc +++ b/tests/ut/cpp/dataset/c_api_datasets_test.cc @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -31,7 +31,7 @@ TEST_F(MindDataTestPipeline, TestCelebADataset) { // Create a CelebA Dataset std::string folder_path = datasets_root_path_ + "/testCelebAData/"; - std::shared_ptr ds = CelebA(folder_path, "all", SequentialSampler(0, 2), false, {}); + std::shared_ptr ds = CelebA(folder_path, "all", std::make_shared(0, 2), false, {}); EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset @@ -208,7 +208,7 @@ TEST_F(MindDataTestPipeline, TestImageFolderFailWithWrongExtensionFail) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 2), {".JGP"}); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 2), {".JGP"}); EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset @@ -260,7 +260,7 @@ TEST_F(MindDataTestPipeline, TestImageFolderFailWithWrongSamplerFail) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, SequentialSampler(-2, 5)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(-2, 5)); EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset @@ -274,7 +274,7 @@ TEST_F(MindDataTestPipeline, TestMnistGetDatasetSize) { // Create a Mnist Dataset std::string folder_path = datasets_root_path_ + "/testMnistData/"; - std::shared_ptr ds = Mnist(folder_path, "all", RandomSampler(false, 20)); + std::shared_ptr ds = Mnist(folder_path, "all", std::make_shared(false, 20)); EXPECT_NE(ds, nullptr); EXPECT_EQ(ds->GetDatasetSize(), 20); } @@ -283,7 +283,7 @@ TEST_F(MindDataTestPipeline, TestMnistFailWithWrongDatasetDirFail) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestMnistFailWithWrongDatasetDirFail."; // Create a Mnist Dataset - std::shared_ptr ds = Mnist("", "all", RandomSampler(false, 10)); + std::shared_ptr ds = Mnist("", "all", std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset @@ -311,7 +311,7 @@ TEST_F(MindDataTestPipeline, TestImageFolderClassIndexDatasetSize) { std::map class_index; class_index["class1"] = 111; class_index["class2"] = 333; - auto ds = ImageFolder(folder_path, false, RandomSampler(), {}, class_index); + auto ds = ImageFolder(folder_path, false, std::make_shared(), {}, class_index); EXPECT_EQ(ds->GetNumClasses(), 2); } @@ -320,6 +320,6 @@ TEST_F(MindDataTestPipeline, TestImageFolderClassIndexDatasetSizeFail) { std::map class_index; class_index["class1"] = 111; class_index["wrong class"] = 333; - auto ds = ImageFolder(folder_path, false, RandomSampler(), {}, class_index); + auto ds = ImageFolder(folder_path, false, std::make_shared(), {}, class_index); EXPECT_EQ(ds->GetNumClasses(), -1); } diff --git a/tests/ut/cpp/dataset/c_api_epoch_ctrl_test.cc b/tests/ut/cpp/dataset/c_api_epoch_ctrl_test.cc index 29e3e8662b..eb213c9e91 100644 --- a/tests/ut/cpp/dataset/c_api_epoch_ctrl_test.cc +++ b/tests/ut/cpp/dataset/c_api_epoch_ctrl_test.cc @@ -34,7 +34,7 @@ TEST_F(MindDataTestEpochCtrl, TestAutoInjectEpoch) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, SequentialSampler(0, sampler_size)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(0, sampler_size)); ds = ds->SetNumWorkers(2); // Create an iterator over the result of the above dataset @@ -79,7 +79,7 @@ TEST_F(MindDataTestEpochCtrl, TestEpoch) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(0, sampler_size)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(0, sampler_size)); ds = ds->SetNumWorkers(3); // Create an iterator over the result of the above dataset @@ -125,7 +125,7 @@ TEST_F(MindDataTestEpochCtrl, TestRepeatEpoch) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(0, sampler_size)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(0, sampler_size)); ds = ds->SetNumWorkers(3); ds = ds->Repeat(num_repeats); @@ -172,7 +172,7 @@ TEST_F(MindDataTestEpochCtrl, TestRepeatRepeatEpoch) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, SequentialSampler(5, sampler_size)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(5, sampler_size)); ds = ds->Repeat(num_repeats[0]); ds = ds->Repeat(num_repeats[1]); diff --git a/tests/ut/cpp/dataset/c_api_samplers_test.cc b/tests/ut/cpp/dataset/c_api_samplers_test.cc index 8a0b7a557e..e0060eae89 100644 --- a/tests/ut/cpp/dataset/c_api_samplers_test.cc +++ b/tests/ut/cpp/dataset/c_api_samplers_test.cc @@ -16,6 +16,7 @@ #include "common/common.h" #include "minddata/dataset/engine/datasetops/source/sampler/sampler.h" #include "minddata/dataset/include/datasets.h" +#include using namespace mindspore::dataset; using mindspore::dataset::Tensor; @@ -25,27 +26,27 @@ class MindDataTestPipeline : public UT::DatasetOpTesting { }; TEST_F(MindDataTestPipeline, TestImageFolderWithSamplers) { - std::shared_ptr sampl = DistributedSampler(2, 1); + std::shared_ptr sampl = std::make_shared(2, 1); EXPECT_NE(sampl, nullptr); - sampl = PKSampler(3); + sampl = std::make_shared(3); EXPECT_NE(sampl, nullptr); - sampl = RandomSampler(false, 12); + sampl = std::make_shared(false, 12); EXPECT_NE(sampl, nullptr); - sampl = SequentialSampler(0, 12); + sampl = std::make_shared(0, 12); EXPECT_NE(sampl, nullptr); std::vector weights = {0.9, 0.8, 0.68, 0.7, 0.71, 0.6, 0.5, 0.4, 0.3, 0.5, 0.2, 0.1}; - sampl = WeightedRandomSampler(weights, 12); + sampl = std::make_shared(weights, 12); EXPECT_NE(sampl, nullptr); std::vector indices = {1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23}; - sampl = SubsetSampler(indices); + sampl = std::make_shared(indices); EXPECT_NE(sampl, nullptr); - sampl = SubsetRandomSampler(indices); + sampl = std::make_shared(indices); EXPECT_NE(sampl, nullptr); // Create an ImageFolder Dataset @@ -88,47 +89,47 @@ TEST_F(MindDataTestPipeline, TestImageFolderWithSamplers) { TEST_F(MindDataTestPipeline, TestCalculateNumSamples) { int64_t num_rows = 30; // dummy variable for number of rows in the dataset - std::shared_ptr sampl = DistributedSampler(2, 1, false, 6); + std::shared_ptr sampl = std::make_shared(2, 1, false, 6, 1, -1, true); EXPECT_NE(sampl, nullptr); std::shared_ptr sampler_rt; sampl->SamplerBuild(&sampler_rt); EXPECT_EQ(sampler_rt->CalculateNumSamples(num_rows), 6); - sampl = PKSampler(3, false); + sampl = std::make_shared(3, false, 0); EXPECT_NE(sampl, nullptr); sampl->SamplerBuild(&sampler_rt); EXPECT_EQ(sampler_rt->CalculateNumSamples(num_rows), 30); - sampl = RandomSampler(false, 12); + sampl = std::make_shared(false, 12); EXPECT_NE(sampl, nullptr); sampl->SamplerBuild(&sampler_rt); EXPECT_EQ(sampler_rt->CalculateNumSamples(num_rows), 12); - sampl = SequentialSampler(0, 10); + sampl = std::make_shared(0, 10); EXPECT_NE(sampl, nullptr); sampl->SamplerBuild(&sampler_rt); EXPECT_EQ(sampler_rt->CalculateNumSamples(num_rows), 10); std::vector weights = {0.9, 0.8, 0.68, 0.7, 0.71, 0.6, 0.5, 0.4, 0.3, 0.5, 0.2, 0.1}; - sampl = WeightedRandomSampler(weights, 12); + sampl = std::make_shared(weights, 12); EXPECT_NE(sampl, nullptr); sampl->SamplerBuild(&sampler_rt); EXPECT_EQ(sampler_rt->CalculateNumSamples(num_rows), 12); std::vector indices = {1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21}; - sampl = SubsetRandomSampler(indices, 11); + sampl = std::make_shared(indices, 11); EXPECT_NE(sampl, nullptr); sampl->SamplerBuild(&sampler_rt); EXPECT_EQ(sampler_rt->CalculateNumSamples(num_rows), 11); // Testing chains // Parent and child have num_samples - std::shared_ptr sampl1 = WeightedRandomSampler(weights, 12); + std::shared_ptr sampl1 = std::make_shared(weights, 12); EXPECT_NE(sampl1, nullptr); std::shared_ptr sampler_rt1; sampl1->SamplerBuild(&sampler_rt1); - std::shared_ptr sampl2 = SequentialSampler(0, 10); + std::shared_ptr sampl2 = std::make_shared(0, 10); EXPECT_NE(sampl2, nullptr); std::shared_ptr sampler_rt2; sampl2->SamplerBuild(&sampler_rt2); @@ -136,12 +137,12 @@ TEST_F(MindDataTestPipeline, TestCalculateNumSamples) { EXPECT_EQ(sampler_rt2->CalculateNumSamples(num_rows), 10); // Parent doesn't have num_samples - std::shared_ptr sampl3 = WeightedRandomSampler(weights, 12); + std::shared_ptr sampl3 = std::make_shared(weights, 12); EXPECT_NE(sampl3, nullptr); std::shared_ptr sampler_rt3; sampl3->SamplerBuild(&sampler_rt3); - std::shared_ptr sampl4 = SubsetRandomSampler(indices); + std::shared_ptr sampl4 = std::make_shared(indices, 0); EXPECT_NE(sampl4, nullptr); std::shared_ptr sampler_rt4; sampl4->SamplerBuild(&sampler_rt4); @@ -149,12 +150,12 @@ TEST_F(MindDataTestPipeline, TestCalculateNumSamples) { EXPECT_EQ(sampler_rt4->CalculateNumSamples(num_rows), 11); // Child doesn't have num_samples - std::shared_ptr sampl5 = RandomSampler(false); + std::shared_ptr sampl5 = std::make_shared(false, 0); EXPECT_NE(sampl5, nullptr); std::shared_ptr sampler_rt5; sampl5->SamplerBuild(&sampler_rt5); - std::shared_ptr sampl6 = PKSampler(3, false, 7); + std::shared_ptr sampl6 = std::make_shared(3, false, 7); EXPECT_NE(sampl6, nullptr); std::shared_ptr sampler_rt6; sampl6->SamplerBuild(&sampler_rt6); @@ -164,41 +165,80 @@ TEST_F(MindDataTestPipeline, TestCalculateNumSamples) { TEST_F(MindDataTestPipeline, TestSamplersMoveParameters) { std::vector indices = {1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23}; - std::shared_ptr sampl1 = SubsetRandomSampler(indices); + std::shared_ptr sampl1 = std::make_shared(indices, 0); EXPECT_FALSE(indices.empty()); std::shared_ptr sampler_rt = nullptr; sampl1->SamplerBuild(&sampler_rt); EXPECT_NE(sampler_rt, nullptr); - std::shared_ptr sampl2 = SubsetRandomSampler(std::move(indices)); + std::shared_ptr sampl2 = std::make_shared(std::move(indices), 0); EXPECT_TRUE(indices.empty()); std::shared_ptr sampler_rt2 = nullptr; sampl2->SamplerBuild(&sampler_rt2); EXPECT_NE(sampler_rt, nullptr); } -TEST_F(MindDataTestPipeline, TestWeightedRandomSamplerFail) { - // weights is empty - std::vector weights1 = {}; - std::shared_ptr sampl1 = WeightedRandomSampler(weights1); - EXPECT_EQ(sampl1, nullptr); - - // weights has negative number - std::vector weights2 = {0.5, 0.2, -0.4}; - std::shared_ptr sampl2 = WeightedRandomSampler(weights2); - EXPECT_EQ(sampl2, nullptr); - - // weights elements are all zero - std::vector weights3 = {0, 0, 0}; - std::shared_ptr sampl3 = WeightedRandomSampler(weights3); - EXPECT_EQ(sampl3, nullptr); +TEST_F(MindDataTestPipeline, TestNoSamplerSuccess1) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestNoSamplerSuccess1."; + // Test building a dataset with no sampler provided (defaults to random sampler + + // Create an ImageFolder Dataset + std::string folder_path = datasets_root_path_ + "/testPK/data/"; + std::shared_ptr ds = ImageFolder(folder_path, false); + EXPECT_NE(ds, nullptr); + + // Iterate the dataset and get each row + std::shared_ptr iter = ds->CreateIterator(); + EXPECT_NE(iter, nullptr); + std::unordered_map row; + iter->GetNextRow(&row); + + uint64_t i = 0; + while (row.size() != 0) { + i++; + auto label = row["label"]; + iter->GetNextRow(&row); + } + + EXPECT_EQ(i, ds->GetDatasetSize()); + iter->Stop(); +} + +TEST_F(MindDataTestPipeline, TestDistributedSamplerSuccess1) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDistributedSamplerSuccess1."; + // Test basic setting of distributed_sampler + + // num_shards=4, shard_id=0, shuffle=false, num_samplers=0, seed=0, offset=-1, even_dist=true + std::shared_ptr sampler = std::make_shared(4, 0, false, 0, 0, -1, true); + EXPECT_NE(sampler, nullptr); + + // Create an ImageFolder Dataset + std::string folder_path = datasets_root_path_ + "/testPK/data/"; + std::shared_ptr ds = ImageFolder(folder_path, false, sampler); + EXPECT_NE(ds, nullptr); + + // Iterate the dataset and get each row + std::shared_ptr iter = ds->CreateIterator(); + EXPECT_NE(iter, nullptr); + std::unordered_map row; + iter->GetNextRow(&row); + + uint64_t i = 0; + while (row.size() != 0) { + i++; + auto label = row["label"]; + iter->GetNextRow(&row); + } + + EXPECT_EQ(i, 11); + iter->Stop(); } -TEST_F(MindDataTestPipeline, TestDistributedSamplerSuccess) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDistributedSamplerSuccess."; +TEST_F(MindDataTestPipeline, TestDistributedSamplerSuccess2) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDistributedSamplerSuccess2."; // Test basic setting of distributed_sampler // num_shards=4, shard_id=0, shuffle=false, num_samplers=0, seed=0, offset=-1, even_dist=true - std::shared_ptr sampler = DistributedSampler(4, 0, false, 0, 0, -1, true); + Sampler *sampler = new DistributedSampler(4, 0, false, 0, 0, -1, true); EXPECT_NE(sampler, nullptr); // Create an ImageFolder Dataset @@ -223,16 +263,102 @@ TEST_F(MindDataTestPipeline, TestDistributedSamplerSuccess) { iter->Stop(); } +TEST_F(MindDataTestPipeline, TestDistributedSamplerSuccess3) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDistributedSamplerSuccess3."; + // Test basic setting of distributed_sampler + + // num_shards=4, shard_id=0, shuffle=false, num_samplers=0, seed=0, offset=-1, even_dist=true + DistributedSampler sampler = DistributedSampler(4, 0, false, 0, 0, -1, true); + + // Create an ImageFolder Dataset + std::string folder_path = datasets_root_path_ + "/testPK/data/"; + std::shared_ptr ds = ImageFolder(folder_path, false, sampler); + EXPECT_NE(ds, nullptr); + + // Iterate the dataset and get each row + std::shared_ptr iter = ds->CreateIterator(); + EXPECT_NE(iter, nullptr); + std::unordered_map row; + iter->GetNextRow(&row); + + uint64_t i = 0; + while (row.size() != 0) { + i++; + auto label = row["label"]; + iter->GetNextRow(&row); + } + + EXPECT_EQ(i, 11); + iter->Stop(); +} + +TEST_F(MindDataTestPipeline, TestDistributedSamplerFail1) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDistributedSamplerFail1."; + // Test basic setting of distributed_sampler + + // num_shards=4, shard_id=0, shuffle=false, num_samplers=0, seed=0, offset=5, even_dist=true + // offset=5 which is greater than num_shards=4 --> will fail later + std::shared_ptr sampler = std::make_shared(4, 0, false, 0, 0, 5, false); + EXPECT_NE(sampler, nullptr); + + // Create an ImageFolder Dataset + std::string folder_path = datasets_root_path_ + "/testPK/data/"; + std::shared_ptr ds = ImageFolder(folder_path, false, sampler); + EXPECT_NE(ds, nullptr); + + // Iterate will fail because sampler is not initiated successfully. + std::shared_ptr iter = ds->CreateIterator(); + EXPECT_EQ(iter, nullptr); +} + +TEST_F(MindDataTestPipeline, TestDistributedSamplerFail2) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDistributedSamplerFail2."; + // Test basic setting of distributed_sampler + + // num_shards=4, shard_id=0, shuffle=false, num_samplers=0, seed=0, offset=5, even_dist=true + // offset=5 which is greater than num_shards=4 --> will fail later + Sampler *sampler = new DistributedSampler(4, 0, false, 0, 0, 5, false); + EXPECT_NE(sampler, nullptr); + + // Create an ImageFolder Dataset + std::string folder_path = datasets_root_path_ + "/testPK/data/"; + std::shared_ptr ds = ImageFolder(folder_path, false, sampler); + EXPECT_NE(ds, nullptr); + + // Iterate will fail because sampler is not initiated successfully. + std::shared_ptr iter = ds->CreateIterator(); + EXPECT_EQ(iter, nullptr); +} + +TEST_F(MindDataTestPipeline, TestDistributedSamplerFail3) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDistributedSamplerFail3."; + // Test basic setting of distributed_sampler + + // num_shards=4, shard_id=0, shuffle=false, num_samplers=0, seed=0, offset=5, even_dist=true + // offset=5 which is greater than num_shards=4 --> will fail later + DistributedSampler sampler = DistributedSampler(4, 0, false, 0, 0, 5, false); + + // Create an ImageFolder Dataset + std::string folder_path = datasets_root_path_ + "/testPK/data/"; + std::shared_ptr ds = ImageFolder(folder_path, false, sampler); + EXPECT_NE(ds, nullptr); + + // Iterate will fail because sampler is not initiated successfully. + std::shared_ptr iter = ds->CreateIterator(); + EXPECT_EQ(iter, nullptr); +} + TEST_F(MindDataTestPipeline, TestSamplerAddChild) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSamplerAddChild."; - auto sampler = DistributedSampler(1, 0, false, 5, 0, -1, true); + auto sampler = std::make_shared(1, 0, false, 5, 0, -1, true); EXPECT_NE(sampler, nullptr); - auto child_sampler = SequentialSampler(); - sampler->AddChildSampler(child_sampler); + auto child_sampler = std::make_shared(); EXPECT_NE(child_sampler, nullptr); + sampler->AddChild(child_sampler); + // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::shared_ptr ds = ImageFolder(folder_path, false, sampler); @@ -253,12 +379,3 @@ TEST_F(MindDataTestPipeline, TestSamplerAddChild) { EXPECT_EQ(ds->GetDatasetSize(), 5); iter->Stop(); } - -TEST_F(MindDataTestPipeline, TestDistributedSamplerFail) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDistributedSamplerFail."; - // Test invalid offset setting of distributed_sampler - - // offset=5 which is greater than num_shards=4 - std::shared_ptr sampler = DistributedSampler(4, 0, false, 0, 0, 5, false); - EXPECT_EQ(sampler, nullptr); -} diff --git a/tests/ut/cpp/dataset/c_api_transforms_test.cc b/tests/ut/cpp/dataset/c_api_transforms_test.cc index c453ac6d40..193df824ea 100644 --- a/tests/ut/cpp/dataset/c_api_transforms_test.cc +++ b/tests/ut/cpp/dataset/c_api_transforms_test.cc @@ -33,7 +33,7 @@ TEST_F(MindDataTestPipeline, TestComposeSuccess) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, false, RandomSampler(false, 3)); + std::shared_ptr ds = ImageFolder(folder_path, false, std::make_shared(false, 3)); EXPECT_NE(ds, nullptr); // Create objects for the tensor ops @@ -68,6 +68,7 @@ TEST_F(MindDataTestPipeline, TestComposeSuccess) { EXPECT_EQ(i, 3); + // Manually terminate the pipeline iter->Stop(); } @@ -77,7 +78,7 @@ TEST_F(MindDataTestPipeline, TestComposeFail1) { // Create a Cifar10 Dataset std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); + std::shared_ptr ds = Cifar10(folder_path, "all", std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Resize: Non-positive size value: -1 at element: 0 @@ -100,7 +101,7 @@ TEST_F(MindDataTestPipeline, TestComposeFail2) { // Create a Cifar10 Dataset std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); + std::shared_ptr ds = Cifar10(folder_path, "all", std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Compose: transform ops must not be null @@ -121,7 +122,7 @@ TEST_F(MindDataTestPipeline, TestComposeFail3) { // Create a Cifar10 Dataset std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); + std::shared_ptr ds = Cifar10(folder_path, "all", std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Compose: transform list must not be empty @@ -142,7 +143,7 @@ TEST_F(MindDataTestPipeline, TestDuplicateSuccess) { // Create a Cifar10 Dataset std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); + std::shared_ptr ds = Cifar10(folder_path, "all", std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create objects for the tensor ops @@ -182,7 +183,7 @@ TEST_F(MindDataTestPipeline, TestOneHotSuccess1) { // Create a Cifar10 Dataset std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; int number_of_classes = 10; - std::shared_ptr ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); + std::shared_ptr ds = Cifar10(folder_path, "all", std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create objects for the tensor ops @@ -248,7 +249,7 @@ TEST_F(MindDataTestPipeline, TestOneHotSuccess1) { TEST_F(MindDataTestPipeline, TestOneHotSuccess2) { // Create a Cifar10 Dataset std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); + std::shared_ptr ds = Cifar10(folder_path, "all", std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a Batch operation on ds @@ -299,7 +300,7 @@ TEST_F(MindDataTestPipeline, TestOneHotFail1) { // Create a Cifar10 Dataset std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); + std::shared_ptr ds = Cifar10(folder_path, "all", std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // incorrect num_class @@ -320,7 +321,7 @@ TEST_F(MindDataTestPipeline, TestOneHotFail2) { // Create a Cifar10 Dataset std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); + std::shared_ptr ds = Cifar10(folder_path, "all", std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // incorrect num_class @@ -341,7 +342,7 @@ TEST_F(MindDataTestPipeline, TestRandomApplySuccess) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 5)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 5)); EXPECT_NE(ds, nullptr); // Create objects for the tensor ops @@ -382,7 +383,7 @@ TEST_F(MindDataTestPipeline, TestRandomApplyFail1) { // Create a Cifar10 Dataset std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); + std::shared_ptr ds = Cifar10(folder_path, "all", std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Resize: Non-positive size value: -1 at element: 0 @@ -405,7 +406,7 @@ TEST_F(MindDataTestPipeline, TestRandomApplyFail2) { // Create a Cifar10 Dataset std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); + std::shared_ptr ds = Cifar10(folder_path, "all", std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // RandomApply: transform ops must not be null @@ -426,7 +427,7 @@ TEST_F(MindDataTestPipeline, TestRandomApplyFail3) { // Create a Cifar10 Dataset std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); + std::shared_ptr ds = Cifar10(folder_path, "all", std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // RandomApply: Probability has to be between 0 and 1 @@ -447,7 +448,7 @@ TEST_F(MindDataTestPipeline, TestRandomApplyFail4) { // Create a Cifar10 Dataset std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); + std::shared_ptr ds = Cifar10(folder_path, "all", std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // RandomApply: transform list must not be empty @@ -468,7 +469,7 @@ TEST_F(MindDataTestPipeline, TestRandomChoiceSuccess) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 3)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 3)); EXPECT_NE(ds, nullptr); // Create objects for the tensor ops @@ -510,7 +511,8 @@ TEST_F(MindDataTestPipeline, TestRandomChoiceFail1) { // Create a Cifar10 Dataset std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); + RandomSampler sampler = RandomSampler(false, 10); + std::shared_ptr ds = Cifar10(folder_path, "all", sampler); EXPECT_NE(ds, nullptr); // Resize: Non-positive size value: -1 at element: 0 @@ -533,7 +535,7 @@ TEST_F(MindDataTestPipeline, TestRandomChoiceFail2) { // Create a Cifar10 Dataset std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); + std::shared_ptr ds = Cifar10(folder_path, "all", std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // RandomChoice: transform ops must not be null @@ -554,7 +556,7 @@ TEST_F(MindDataTestPipeline, TestRandomChoiceFail3) { // Create a Cifar10 Dataset std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); + std::shared_ptr ds = Cifar10(folder_path, "all", std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // RandomChoice: transform list must not be empty @@ -575,7 +577,7 @@ TEST_F(MindDataTestPipeline, TestTypeCastSuccess) { // Create a Cifar10 Dataset std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds = Cifar10(folder_path, "all", RandomSampler(false, 1)); + std::shared_ptr ds = Cifar10(folder_path, "all", std::make_shared(false, 1)); EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset @@ -625,7 +627,7 @@ TEST_F(MindDataTestPipeline, TestTypeCastFail) { // Create a Cifar10 Dataset std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); + std::shared_ptr ds = Cifar10(folder_path, "all", std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // incorrect data type diff --git a/tests/ut/cpp/dataset/c_api_vision_a_to_q_test.cc b/tests/ut/cpp/dataset/c_api_vision_a_to_q_test.cc index 4927989053..3f86dfb5dc 100644 --- a/tests/ut/cpp/dataset/c_api_vision_a_to_q_test.cc +++ b/tests/ut/cpp/dataset/c_api_vision_a_to_q_test.cc @@ -32,7 +32,7 @@ TEST_F(MindDataTestPipeline, TestAutoContrastSuccess1) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 5)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 5)); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds @@ -81,7 +81,7 @@ TEST_F(MindDataTestPipeline, TestAutoContrastSuccess2) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 5)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 5)); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds @@ -142,7 +142,7 @@ TEST_F(MindDataTestPipeline, TestCenterCrop) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 5)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 5)); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds @@ -228,7 +228,7 @@ TEST_F(MindDataTestPipeline, TestCutMixBatchSuccess1) { // Create a Cifar10 Dataset std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; int number_of_classes = 10; - std::shared_ptr ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); + std::shared_ptr ds = Cifar10(folder_path, "all", std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create objects for the tensor ops @@ -299,7 +299,7 @@ TEST_F(MindDataTestPipeline, TestCutMixBatchSuccess2) { // Create a Cifar10 Dataset std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; int number_of_classes = 10; - std::shared_ptr ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); + std::shared_ptr ds = Cifar10(folder_path, "all", std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a Batch operation on ds @@ -358,7 +358,7 @@ TEST_F(MindDataTestPipeline, TestCutMixBatchFail1) { // Create a Cifar10 Dataset std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); + std::shared_ptr ds = Cifar10(folder_path, "all", std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a Batch operation on ds @@ -393,7 +393,7 @@ TEST_F(MindDataTestPipeline, TestCutMixBatchFail2) { // Create a Cifar10 Dataset std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); + std::shared_ptr ds = Cifar10(folder_path, "all", std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a Batch operation on ds @@ -428,7 +428,7 @@ TEST_F(MindDataTestPipeline, TestCutMixBatchFail3) { // Create a Cifar10 Dataset std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); + std::shared_ptr ds = Cifar10(folder_path, "all", std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a Batch operation on ds @@ -462,7 +462,7 @@ TEST_F(MindDataTestPipeline, TestCutMixBatchFail4) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCutMixBatchFail4 with invalid greater than 1 prob parameter."; std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); + std::shared_ptr ds = Cifar10(folder_path, "all", std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a Batch operation on ds @@ -521,7 +521,7 @@ TEST_F(MindDataTestPipeline, TestCutOut) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds @@ -573,7 +573,7 @@ TEST_F(MindDataTestPipeline, TestDecode) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, false, RandomSampler(false, 10)); + std::shared_ptr ds = ImageFolder(folder_path, false, std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds @@ -620,7 +620,7 @@ TEST_F(MindDataTestPipeline, TestHwcToChw) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds @@ -672,7 +672,7 @@ TEST_F(MindDataTestPipeline, TestInvert) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 20)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 20)); EXPECT_NE(ds, nullptr); // Create objects for the tensor ops @@ -710,7 +710,7 @@ TEST_F(MindDataTestPipeline, TestMixUpBatchFail1) { // FIXME: For error tests, need to check for failure from CreateIterator execution // Create a Cifar10 Dataset std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); + std::shared_ptr ds = Cifar10(folder_path, "all", std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a Batch operation on ds @@ -744,7 +744,7 @@ TEST_F(MindDataTestPipeline, TestMixUpBatchFail2) { // FIXME: For error tests, need to check for failure from CreateIterator execution // Create a Cifar10 Dataset std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); + std::shared_ptr ds = Cifar10(folder_path, "all", std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a Batch operation on ds @@ -778,7 +778,7 @@ TEST_F(MindDataTestPipeline, TestMixUpBatchSuccess1) { // Create a Cifar10 Dataset std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); + std::shared_ptr ds = Cifar10(folder_path, "all", std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a Batch operation on ds @@ -829,7 +829,7 @@ TEST_F(MindDataTestPipeline, TestMixUpBatchSuccess2) { // Create a Cifar10 Dataset std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); + std::shared_ptr ds = Cifar10(folder_path, "all", std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a Batch operation on ds @@ -880,7 +880,7 @@ TEST_F(MindDataTestPipeline, TestNormalize) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds @@ -958,7 +958,7 @@ TEST_F(MindDataTestPipeline, TestNormalizePad) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds @@ -1025,7 +1025,7 @@ TEST_F(MindDataTestPipeline, TestPad) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds diff --git a/tests/ut/cpp/dataset/c_api_vision_bounding_box_augment_test.cc b/tests/ut/cpp/dataset/c_api_vision_bounding_box_augment_test.cc index c7a2cd754b..1d4cf58ba0 100644 --- a/tests/ut/cpp/dataset/c_api_vision_bounding_box_augment_test.cc +++ b/tests/ut/cpp/dataset/c_api_vision_bounding_box_augment_test.cc @@ -30,7 +30,7 @@ TEST_F(MindDataTestPipeline, TestBoundingBoxAugmentSuccess) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestBoundingBoxAugmentSuccess."; // Create an VOC Dataset std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; - std::shared_ptr ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 3)); + std::shared_ptr ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared(0, 3)); EXPECT_NE(ds, nullptr); /* FIXME - Resolve BoundingBoxAugment to properly handle TensorTransform input diff --git a/tests/ut/cpp/dataset/c_api_vision_r_to_z_test.cc b/tests/ut/cpp/dataset/c_api_vision_r_to_z_test.cc index 8cab5c5f93..fc6f9855af 100644 --- a/tests/ut/cpp/dataset/c_api_vision_r_to_z_test.cc +++ b/tests/ut/cpp/dataset/c_api_vision_r_to_z_test.cc @@ -30,7 +30,7 @@ TEST_F(MindDataTestPipeline, TestRescaleSucess1) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRescaleSucess1."; // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, SequentialSampler(0, 1)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(0, 1)); EXPECT_NE(ds, nullptr); // Create an iterator over the result of the above dataset @@ -76,7 +76,7 @@ TEST_F(MindDataTestPipeline, TestRescaleSucess2) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRescaleSucess2 with different params."; // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 1)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 1)); EXPECT_NE(ds, nullptr); // Create objects for the tensor ops @@ -121,7 +121,7 @@ TEST_F(MindDataTestPipeline, TestResize1) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestResize1 with single integer input."; // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 6)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 6)); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds @@ -183,7 +183,7 @@ TEST_F(MindDataTestPipeline, TestResizeWithBBoxSuccess) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestResizeWithBBoxSuccess."; // Create an VOC Dataset std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; - std::shared_ptr ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 3)); + std::shared_ptr ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared(0, 3)); EXPECT_NE(ds, nullptr); // Create objects for the tensor ops diff --git a/tests/ut/cpp/dataset/c_api_vision_random_subselect_policy_test.cc b/tests/ut/cpp/dataset/c_api_vision_random_subselect_policy_test.cc index 08b6e17f6a..8c75ea9ac2 100644 --- a/tests/ut/cpp/dataset/c_api_vision_random_subselect_policy_test.cc +++ b/tests/ut/cpp/dataset/c_api_vision_random_subselect_policy_test.cc @@ -31,7 +31,7 @@ TEST_F(MindDataTestPipeline, TestRandomSelectSubpolicySuccess) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 7)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 7)); EXPECT_NE(ds, nullptr); /* FIXME - Resolve RandomSelectSubpolicy to properly handle TensorTransform input diff --git a/tests/ut/cpp/dataset/c_api_vision_random_test.cc b/tests/ut/cpp/dataset/c_api_vision_random_test.cc index 7e8f5c10cc..f3284fbb1f 100644 --- a/tests/ut/cpp/dataset/c_api_vision_random_test.cc +++ b/tests/ut/cpp/dataset/c_api_vision_random_test.cc @@ -46,7 +46,7 @@ TEST_F(MindDataTestPipeline, TestRandomAffineSuccess1) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds @@ -96,7 +96,7 @@ TEST_F(MindDataTestPipeline, TestRandomAffineSuccess2) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds @@ -145,7 +145,7 @@ TEST_F(MindDataTestPipeline, TestRandomColor) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds @@ -209,7 +209,7 @@ TEST_F(MindDataTestPipeline, TestRandomColorAdjust) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds @@ -303,7 +303,7 @@ TEST_F(MindDataTestPipeline, TestRandomCropSuccess) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomCropSuccess."; // Create an VOC Dataset std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; - std::shared_ptr ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 10)); + std::shared_ptr ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared(0, 10)); EXPECT_NE(ds, nullptr); // Create objects for the tensor ops @@ -366,7 +366,7 @@ TEST_F(MindDataTestPipeline, TestRandomCropFail) { // FIXME: For error tests, need to check for failure from CreateIterator execution // Create an VOC Dataset std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; - std::shared_ptr ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 3)); + std::shared_ptr ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared(0, 3)); EXPECT_NE(ds, nullptr); // Create objects for the tensor ops @@ -406,7 +406,7 @@ TEST_F(MindDataTestPipeline, TestRandomCropWithBboxSuccess) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomCropWithBboxSuccess."; // Create an VOC Dataset std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; - std::shared_ptr ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 3)); + std::shared_ptr ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared(0, 3)); EXPECT_NE(ds, nullptr); // Create objects for the tensor ops @@ -446,7 +446,7 @@ TEST_F(MindDataTestPipeline, TestRandomCropWithBboxFail) { // FIXME: For error tests, need to check for failure from CreateIterator execution // Create an VOC Dataset std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; - std::shared_ptr ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 3)); + std::shared_ptr ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared(0, 3)); EXPECT_NE(ds, nullptr); // Create objects for the tensor ops @@ -493,7 +493,7 @@ TEST_F(MindDataTestPipeline, TestRandomHorizontalFlipWithBBoxSuccess) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomHorizontalFlipWithBBoxSuccess."; // Create an VOC Dataset std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; - std::shared_ptr ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 3)); + std::shared_ptr ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared(0, 3)); EXPECT_NE(ds, nullptr); // Create objects for the tensor ops @@ -531,7 +531,7 @@ TEST_F(MindDataTestPipeline, TestRandomHorizontalFlipWithBBoxFail) { // FIXME: For error tests, need to check for failure from CreateIterator execution // Create an VOC Dataset std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; - std::shared_ptr ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 3)); + std::shared_ptr ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared(0, 3)); EXPECT_NE(ds, nullptr); // Create objects for the tensor ops @@ -548,7 +548,7 @@ TEST_F(MindDataTestPipeline, TestRandomHorizontalAndVerticalFlip) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds @@ -618,7 +618,7 @@ TEST_F(MindDataTestPipeline, TestRandomPosterizeSuccess1) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds @@ -667,7 +667,7 @@ TEST_F(MindDataTestPipeline, TestRandomPosterizeSuccess2) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds @@ -716,7 +716,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizeSuccess1) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 5)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 5)); EXPECT_NE(ds, nullptr); // Create objects for the tensor ops @@ -756,7 +756,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizeSuccess2) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 3)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 3)); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds @@ -821,7 +821,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizeWithBBoxSuccess1) { // Create an VOC Dataset std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; - std::shared_ptr ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 3)); + std::shared_ptr ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared(0, 3)); EXPECT_NE(ds, nullptr); // Create objects for the tensor ops @@ -861,7 +861,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizeWithBBoxSuccess2) { // Create an VOC Dataset std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; - std::shared_ptr ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 4)); + std::shared_ptr ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared(0, 4)); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds @@ -921,7 +921,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizedCropSuccess1) { // Testing RandomResizedCrop with default values // Create a Cifar10 Dataset std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); + std::shared_ptr ds = Cifar10(folder_path, "all", std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create objects for the tensor ops @@ -960,7 +960,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizedCropSuccess2) { // Testing RandomResizedCrop with non-default values // Create a Cifar10 Dataset std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); + std::shared_ptr ds = Cifar10(folder_path, "all", std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create objects for the tensor ops @@ -1000,7 +1000,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizedCropFail1) { // This should fail because size has negative value // Create a Cifar10 Dataset std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); + std::shared_ptr ds = Cifar10(folder_path, "all", std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create objects for the tensor ops @@ -1020,7 +1020,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizedCropFail2) { // This should fail because scale isn't in {min, max} format // Create a Cifar10 Dataset std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); + std::shared_ptr ds = Cifar10(folder_path, "all", std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create objects for the tensor ops @@ -1040,7 +1040,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizedCropFail3) { // This should fail because ratio isn't in {min, max} format // Create a Cifar10 Dataset std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); + std::shared_ptr ds = Cifar10(folder_path, "all", std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create objects for the tensor ops @@ -1060,7 +1060,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizedCropFail4) { // This should fail because scale has a size of more than 2 // Create a Cifar10 Dataset std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); + std::shared_ptr ds = Cifar10(folder_path, "all", std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create objects for the tensor ops @@ -1080,7 +1080,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizedCropWithBBoxSuccess1) { // Testing RandomResizedCropWithBBox with default values // Create an VOC Dataset std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; - std::shared_ptr ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 4)); + std::shared_ptr ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared(0, 4)); EXPECT_NE(ds, nullptr); // Create objects for the tensor ops @@ -1119,7 +1119,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizedCropWithBBoxSuccess2) { // Testing RandomResizedCropWithBBox with non-default values // Create an VOC Dataset std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; - std::shared_ptr ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 4)); + std::shared_ptr ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared(0, 4)); EXPECT_NE(ds, nullptr); // Create objects for the tensor ops @@ -1160,7 +1160,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizedCropWithBBoxFail1) { // This should fail because size has negative value // Create a Cifar10 Dataset std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); + std::shared_ptr ds = Cifar10(folder_path, "all", std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create objects for the tensor ops @@ -1173,7 +1173,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizedCropWithBBoxFail2) { // This should fail because scale isn't in {min, max} format // Create a Cifar10 Dataset std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); + std::shared_ptr ds = Cifar10(folder_path, "all", std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create objects for the tensor ops @@ -1186,7 +1186,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizedCropWithBBoxFail3) { // This should fail because ratio isn't in {min, max} format // Create a Cifar10 Dataset std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); + std::shared_ptr ds = Cifar10(folder_path, "all", std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create objects for the tensor ops @@ -1199,7 +1199,7 @@ TEST_F(MindDataTestPipeline, TestRandomResizedCropWithBBoxFail4) { // This should fail because scale has a size of more than 2 // Create a Cifar10 Dataset std::string folder_path = datasets_root_path_ + "/testCifar10Data/"; - std::shared_ptr ds = Cifar10(folder_path, "all", RandomSampler(false, 10)); + std::shared_ptr ds = Cifar10(folder_path, "all", std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create objects for the tensor ops @@ -1212,7 +1212,7 @@ TEST_F(MindDataTestPipeline, TestRandomRotation) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds @@ -1273,7 +1273,7 @@ TEST_F(MindDataTestPipeline, TestRandomRotationFail) { // FIXME: For error tests, need to check for failure from CreateIterator execution // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create objects for the tensor ops @@ -1309,7 +1309,7 @@ TEST_F(MindDataTestPipeline, TestRandomSharpness) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds @@ -1374,7 +1374,7 @@ TEST_F(MindDataTestPipeline, TestRandomSolarizeSucess1) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create objects for the tensor ops @@ -1414,7 +1414,7 @@ TEST_F(MindDataTestPipeline, TestRandomSolarizeSucess2) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, RandomSampler(false, 10)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(false, 10)); EXPECT_NE(ds, nullptr); // Create objects for the tensor ops @@ -1484,7 +1484,7 @@ TEST_F(MindDataTestPipeline, TestRandomVerticalFlipWithBBoxSuccess) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomVerticalFlipWithBBoxSuccess."; // Create an VOC Dataset std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; - std::shared_ptr ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 3)); + std::shared_ptr ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared(0, 3)); EXPECT_NE(ds, nullptr); // Create objects for the tensor ops @@ -1522,7 +1522,7 @@ TEST_F(MindDataTestPipeline, TestRandomVerticalFlipWithBBoxFail) { // FIXME: For error tests, need to check for failure from CreateIterator execution // Create an VOC Dataset std::string folder_path = datasets_root_path_ + "/testVOC2012_2"; - std::shared_ptr ds = VOC(folder_path, "Detection", "train", {}, true, SequentialSampler(0, 3)); + std::shared_ptr ds = VOC(folder_path, "Detection", "train", {}, true, std::make_shared(0, 3)); EXPECT_NE(ds, nullptr); // Create objects for the tensor ops diff --git a/tests/ut/cpp/dataset/c_api_vision_soft_dvpp_test.cc b/tests/ut/cpp/dataset/c_api_vision_soft_dvpp_test.cc index 0b8c1b33e9..5240f713e1 100644 --- a/tests/ut/cpp/dataset/c_api_vision_soft_dvpp_test.cc +++ b/tests/ut/cpp/dataset/c_api_vision_soft_dvpp_test.cc @@ -32,7 +32,7 @@ TEST_F(MindDataTestPipeline, TestSoftDvppDecodeRandomCropResizeJpegSuccess1) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, false, RandomSampler(false, 4)); + std::shared_ptr ds = ImageFolder(folder_path, false, std::make_shared(false, 4)); EXPECT_NE(ds, nullptr); // Create objects for the tensor ops @@ -74,7 +74,7 @@ TEST_F(MindDataTestPipeline, TestSoftDvppDecodeRandomCropResizeJpegSuccess2) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, false, RandomSampler(false, 6)); + std::shared_ptr ds = ImageFolder(folder_path, false, std::make_shared(false, 6)); EXPECT_NE(ds, nullptr); // Create objects for the tensor ops @@ -158,7 +158,7 @@ TEST_F(MindDataTestPipeline, TestSoftDvppDecodeResizeJpegSuccess1) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSoftDvppDecodeResizeJpegSuccess1 with single integer input."; // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, false, RandomSampler(false, 4)); + std::shared_ptr ds = ImageFolder(folder_path, false, std::make_shared(false, 4)); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds @@ -201,7 +201,7 @@ TEST_F(MindDataTestPipeline, TestSoftDvppDecodeResizeJpegSuccess2) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestSoftDvppDecodeResizeJpegSuccess2 with (height, width) input."; // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, false, RandomSampler(false, 2)); + std::shared_ptr ds = ImageFolder(folder_path, false, std::make_shared(false, 2)); EXPECT_NE(ds, nullptr); // Create SoftDvppDecodeResizeJpeg object with single integer input diff --git a/tests/ut/cpp/dataset/c_api_vision_uniform_aug_test.cc b/tests/ut/cpp/dataset/c_api_vision_uniform_aug_test.cc index 77d4a103b8..de22c8b681 100644 --- a/tests/ut/cpp/dataset/c_api_vision_uniform_aug_test.cc +++ b/tests/ut/cpp/dataset/c_api_vision_uniform_aug_test.cc @@ -78,7 +78,7 @@ TEST_F(MindDataTestPipeline, TestUniformAugWithOps) { // Create a Mnist Dataset std::string folder_path = datasets_root_path_ + "/testMnistData/"; - std::shared_ptr ds = Mnist(folder_path, "all", RandomSampler(false, 20)); + std::shared_ptr ds = Mnist(folder_path, "all", std::make_shared(false, 20)); EXPECT_NE(ds, nullptr); // Create a Repeat operation on ds diff --git a/tests/ut/cpp/dataset/ir_tensor_op_fusion_pass_test.cc b/tests/ut/cpp/dataset/ir_tensor_op_fusion_pass_test.cc index 9e6de1f8c7..a657ba3acf 100644 --- a/tests/ut/cpp/dataset/ir_tensor_op_fusion_pass_test.cc +++ b/tests/ut/cpp/dataset/ir_tensor_op_fusion_pass_test.cc @@ -36,7 +36,7 @@ TEST_F(MindDataTestTensorOpFusionPass, RandomCropDecodeResizeDisabled) { MS_LOG(INFO) << "Doing MindDataTestTensorOpFusionPass-RandomCropDecodeResizeDisabled"; std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, false, SequentialSampler(0, 11)); + std::shared_ptr ds = ImageFolder(folder_path, false, std::make_shared(0, 11)); // Create objects for the tensor ops std::shared_ptr decode(new vision::Decode()); @@ -67,7 +67,7 @@ TEST_F(MindDataTestTensorOpFusionPass, RandomCropDecodeResizeEnabled) { MS_LOG(INFO) << "Doing MindDataTestTensorOpFusionPass-RandomCropDecodeResizeEnabled"; std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, false, SequentialSampler(0, 11)); + std::shared_ptr ds = ImageFolder(folder_path, false, std::make_shared(0, 11)); // Create objects for the tensor ops std::shared_ptr decode(new vision::Decode()); diff --git a/tests/ut/cpp/dataset/ir_tree_adapter_test.cc b/tests/ut/cpp/dataset/ir_tree_adapter_test.cc index 1adefed645..a0512d89e5 100644 --- a/tests/ut/cpp/dataset/ir_tree_adapter_test.cc +++ b/tests/ut/cpp/dataset/ir_tree_adapter_test.cc @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -48,7 +48,7 @@ TEST_F(MindDataTestTreeAdapter, TestSimpleTreeAdapter) { // Create a Mnist Dataset std::string folder_path = datasets_root_path_ + "/testMnistData/"; - std::shared_ptr ds = Mnist(folder_path, "all", SequentialSampler(0, 4)); + std::shared_ptr ds = Mnist(folder_path, "all", std::make_shared(0, 4)); EXPECT_NE(ds, nullptr); ds = ds->Batch(2); @@ -83,7 +83,7 @@ TEST_F(MindDataTestTreeAdapter, TestTreeAdapterWithRepeat) { // Create a Mnist Dataset std::string folder_path = datasets_root_path_ + "/testMnistData/"; - std::shared_ptr ds = Mnist(folder_path, "all", SequentialSampler(0, 3)); + std::shared_ptr ds = Mnist(folder_path, "all", std::make_shared(0, 3)); EXPECT_NE(ds, nullptr); ds = ds->Batch(2, false); @@ -115,7 +115,7 @@ TEST_F(MindDataTestTreeAdapter, TestProjectMapTreeAdapter) { // Create an ImageFolder Dataset std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds = ImageFolder(folder_path, true, SequentialSampler(0, 2)); + std::shared_ptr ds = ImageFolder(folder_path, true, std::make_shared(0, 2)); EXPECT_NE(ds, nullptr); // Create objects for the tensor ops diff --git a/tests/ut/cpp/dataset/tree_modifying_function_test.cc b/tests/ut/cpp/dataset/tree_modifying_function_test.cc index f9d430f58e..c79cfe4c77 100644 --- a/tests/ut/cpp/dataset/tree_modifying_function_test.cc +++ b/tests/ut/cpp/dataset/tree_modifying_function_test.cc @@ -49,9 +49,9 @@ TEST_F(MindDataTestTreeModifying, AppendChild) { * ds1 */ std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds1 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); - std::shared_ptr ds2 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); - std::shared_ptr ds6 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); + std::shared_ptr ds1 = ImageFolder(folder_path, false, std::make_shared(0, 11)); + std::shared_ptr ds2 = ImageFolder(folder_path, false, std::make_shared(0, 11)); + std::shared_ptr ds6 = ImageFolder(folder_path, false, std::make_shared(0, 11)); std::shared_ptr ds3 = ds1->Take(10); std::shared_ptr ds4 = ds3->Concat({ds2}); Status rc; @@ -105,9 +105,9 @@ TEST_F(MindDataTestTreeModifying, InsertChildAt01) { * */ std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds1 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); + std::shared_ptr ds1 = ImageFolder(folder_path, false, std::make_shared(0, 11)); std::shared_ptr ds3 = ds1->Take(10); - std::shared_ptr ds5 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); + std::shared_ptr ds5 = ImageFolder(folder_path, false, std::make_shared(0, 11)); std::shared_ptr ds2 = ds5->Repeat(4); std::shared_ptr ds4 = ds3->Concat({ds2}); Status rc; @@ -119,7 +119,7 @@ TEST_F(MindDataTestTreeModifying, InsertChildAt01) { EXPECT_EQ(rc, Status::OK()); // Descend two levels as Compile adds the root node and the epochctrl node on top of ds4 std::shared_ptr ds4_node = ir_tree->RootIRNode()->Children()[0]->Children()[0]; - std::shared_ptr ds6 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); + std::shared_ptr ds6 = ImageFolder(folder_path, false, std::make_shared(0, 11)); std::shared_ptr ds6_to_insert = ds6->IRNode(); std::shared_ptr ds2_node = ds4_node->Children()[1]; rc = ds4_node->InsertChildAt(1, ds6_to_insert); @@ -132,7 +132,7 @@ TEST_F(MindDataTestTreeModifying, InsertChildAt01) { EXPECT_EQ(rc, Status::OK()); // Descend two levels as Compile adds the root node and the epochctrl node on top of ds4 ds4_node = ir_tree->RootIRNode()->Children()[0]->Children()[0]; - ds6 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); + ds6 = ImageFolder(folder_path, false, std::make_shared(0, 11)); ds6_to_insert = ds6->IRNode(); std::shared_ptr ds3_node = ds4_node->Children()[0]; rc = ds4_node->InsertChildAt(0, ds6_to_insert); @@ -145,7 +145,7 @@ TEST_F(MindDataTestTreeModifying, InsertChildAt01) { EXPECT_EQ(rc, Status::OK()); // Descend two levels as Compile adds the root node and the epochctrl node on top of ds4 ds4_node = ir_tree->RootIRNode()->Children()[0]->Children()[0]; - ds6 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); + ds6 = ImageFolder(folder_path, false, std::make_shared(0, 11)); ds6_to_insert = ds6->IRNode(); rc = ds4_node->InsertChildAt(2, ds6_to_insert); EXPECT_EQ(rc, Status::OK()); @@ -165,9 +165,9 @@ TEST_F(MindDataTestTreeModifying, InsertChildAt04) { * */ std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds1 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); + std::shared_ptr ds1 = ImageFolder(folder_path, false, std::make_shared(0, 11)); std::shared_ptr ds3 = ds1->Take(10); - std::shared_ptr ds5 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); + std::shared_ptr ds5 = ImageFolder(folder_path, false, std::make_shared(0, 11)); std::shared_ptr ds2 = ds5->Repeat(4); std::shared_ptr ds4 = ds3->Concat({ds2}); Status rc; @@ -179,7 +179,7 @@ TEST_F(MindDataTestTreeModifying, InsertChildAt04) { EXPECT_EQ(rc, Status::OK()); // Descend two levels as Compile adds the root node and the epochctrl node on top of ds4 std::shared_ptr ds4_node = ir_tree->RootIRNode()->Children()[0]->Children()[0]; - std::shared_ptr ds6 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); + std::shared_ptr ds6 = ImageFolder(folder_path, false, std::make_shared(0, 11)); std::shared_ptr ds6_to_insert = ds6->IRNode(); std::shared_ptr ds3_node = ds4_node->Children()[0]; std::shared_ptr ds2_node = ds4_node->Children()[1]; @@ -193,7 +193,7 @@ TEST_F(MindDataTestTreeModifying, InsertChildAt04) { EXPECT_EQ(rc, Status::OK()); // Descend two levels as Compile adds the root node and the epochctrl node on top of ds4 ds4_node = ir_tree->RootIRNode()->Children()[0]->Children()[0]; - ds6 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); + ds6 = ImageFolder(folder_path, false, std::make_shared(0, 11)); ds6_to_insert = ds6->IRNode(); ds3_node = ds4_node->Children()[0]; ds2_node = ds4_node->Children()[1]; @@ -246,8 +246,8 @@ TEST_F(MindDataTestTreeModifying, InsertAbove01) { */ // Case 1 std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds1 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); - std::shared_ptr ds2 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); + std::shared_ptr ds1 = ImageFolder(folder_path, false, std::make_shared(0, 11)); + std::shared_ptr ds2 = ImageFolder(folder_path, false, std::make_shared(0, 11)); std::shared_ptr ds3 = ds1->Take(10); std::shared_ptr ds4 = ds3->Concat({ds2}); Status rc; @@ -271,8 +271,8 @@ TEST_F(MindDataTestTreeModifying, InsertAbove02) { // Case 2 std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds1 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); - std::shared_ptr ds2 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); + std::shared_ptr ds1 = ImageFolder(folder_path, false, std::make_shared(0, 11)); + std::shared_ptr ds2 = ImageFolder(folder_path, false, std::make_shared(0, 11)); std::shared_ptr ds3 = ds1->Take(10); std::shared_ptr ds4 = ds3 + ds2; Status rc; @@ -296,8 +296,8 @@ TEST_F(MindDataTestTreeModifying, InsertAbove03) { // Case 3 std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds1 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); - std::shared_ptr ds2 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); + std::shared_ptr ds1 = ImageFolder(folder_path, false, std::make_shared(0, 11)); + std::shared_ptr ds2 = ImageFolder(folder_path, false, std::make_shared(0, 11)); std::shared_ptr ds3 = ds1->Take(10); std::shared_ptr ds4 = ds3->Concat({ds2}); Status rc; @@ -357,11 +357,11 @@ TEST_F(MindDataTestTreeModifying, Drop01) { * */ std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds7 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); + std::shared_ptr ds7 = ImageFolder(folder_path, false, std::make_shared(0, 11)); std::shared_ptr ds8 = ds7->Take(20); std::shared_ptr ds9 = ds8->Skip(1); - std::shared_ptr ds3 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); - std::shared_ptr ds2 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); + std::shared_ptr ds3 = ImageFolder(folder_path, false, std::make_shared(0, 11)); + std::shared_ptr ds2 = ImageFolder(folder_path, false, std::make_shared(0, 11)); std::shared_ptr ds4 = ds3->Concat({ds2}); std::shared_ptr ds6 = ds4->Take(13); std::shared_ptr ds10 = ds9 + ds6; @@ -419,11 +419,11 @@ TEST_F(MindDataTestTreeModifying, Drop03) { * */ std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds7 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); + std::shared_ptr ds7 = ImageFolder(folder_path, false, std::make_shared(0, 11)); std::shared_ptr ds8 = ds7->Take(20); std::shared_ptr ds9 = ds8->Skip(1); - std::shared_ptr ds3 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); - std::shared_ptr ds2 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); + std::shared_ptr ds3 = ImageFolder(folder_path, false, std::make_shared(0, 11)); + std::shared_ptr ds2 = ImageFolder(folder_path, false, std::make_shared(0, 11)); std::shared_ptr ds4 = ds3->Concat({ds2}); std::shared_ptr ds6 = ds4->Take(13); std::shared_ptr ds10 = ds9 + ds6; @@ -469,14 +469,14 @@ TEST_F(MindDataTestTreeModifying, Drop04) { * */ std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds7 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); + std::shared_ptr ds7 = ImageFolder(folder_path, false, std::make_shared(0, 11)); std::shared_ptr ds8 = ds7->Take(20); std::shared_ptr ds9 = ds8->Skip(1); - std::shared_ptr ds3 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); - std::shared_ptr ds2 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); + std::shared_ptr ds3 = ImageFolder(folder_path, false, std::make_shared(0, 11)); + std::shared_ptr ds2 = ImageFolder(folder_path, false, std::make_shared(0, 11)); std::shared_ptr ds4 = ds3->Concat({ds2}); - std::shared_ptr ds5 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); - std::shared_ptr ds1 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); + std::shared_ptr ds5 = ImageFolder(folder_path, false, std::make_shared(0, 11)); + std::shared_ptr ds1 = ImageFolder(folder_path, false, std::make_shared(0, 11)); std::shared_ptr ds6 = ds5->Concat({ds4, ds1}); std::shared_ptr ds10 = ds9 + ds6; Status rc; @@ -526,13 +526,13 @@ TEST_F(MindDataTestTreeModifying, Drop05) { * */ std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds7 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); + std::shared_ptr ds7 = ImageFolder(folder_path, false, std::make_shared(0, 11)); std::shared_ptr ds8 = ds7->Take(20); std::shared_ptr ds9 = ds8->Skip(1); - std::shared_ptr ds3 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); + std::shared_ptr ds3 = ImageFolder(folder_path, false, std::make_shared(0, 11)); std::shared_ptr ds4 = ds3->Skip(1); - std::shared_ptr ds5 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); - std::shared_ptr ds1 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); + std::shared_ptr ds5 = ImageFolder(folder_path, false, std::make_shared(0, 11)); + std::shared_ptr ds1 = ImageFolder(folder_path, false, std::make_shared(0, 11)); std::shared_ptr ds6 = ds5->Concat({ds4, ds1}); std::shared_ptr ds10 = ds9 + ds6; Status rc; @@ -581,14 +581,14 @@ TEST_F(MindDataTestTreeModifying, Drop06) { * */ std::string folder_path = datasets_root_path_ + "/testPK/data/"; - std::shared_ptr ds7 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); + std::shared_ptr ds7 = ImageFolder(folder_path, false, std::make_shared(0, 11)); std::shared_ptr ds8 = ds7->Take(20); std::shared_ptr ds9 = ds8->Skip(1); - std::shared_ptr ds3 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); - std::shared_ptr ds2 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); + std::shared_ptr ds3 = ImageFolder(folder_path, false, std::make_shared(0, 11)); + std::shared_ptr ds2 = ImageFolder(folder_path, false, std::make_shared(0, 11)); std::shared_ptr ds4 = ds3->Concat({ds2}); - std::shared_ptr ds5 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); - std::shared_ptr ds1 = ImageFolder(folder_path, false, SequentialSampler(0, 11)); + std::shared_ptr ds5 = ImageFolder(folder_path, false, std::make_shared(0, 11)); + std::shared_ptr ds1 = ImageFolder(folder_path, false, std::make_shared(0, 11)); std::shared_ptr ds6 = ds5->Concat({ds4, ds1}); // ds1 is put after (ds5, ds4)!!! std::shared_ptr ds10 = ds9 + ds6; Status rc;