From 913a49f3af32b97e33bd5244760eb51fd8aaf126 Mon Sep 17 00:00:00 2001 From: alex-yuyue Date: Sun, 25 Oct 2020 01:54:27 -0400 Subject: [PATCH] Add support for C++ API: GetNumClasses Signed-off-by: alex-yuyue --- .../ccsrc/minddata/dataset/api/datasets.cc | 89 +++++++++++++++---- .../dataset/engine/consumers/tree_consumer.cc | 8 ++ .../dataset/engine/consumers/tree_consumer.h | 1 + .../dataset/engine/datasetops/dataset_op.cc | 16 +++- .../dataset/engine/datasetops/dataset_op.h | 5 ++ .../datasetops/source/image_folder_op.cc | 12 +++ .../datasetops/source/image_folder_op.h | 5 ++ .../engine/datasetops/source/manifest_op.cc | 17 ++++ .../engine/datasetops/source/manifest_op.h | 5 ++ .../ccsrc/minddata/dataset/include/datasets.h | 10 ++- .../cpp/dataset/c_api_dataset_album_test.cc | 20 +++++ .../cpp/dataset/c_api_dataset_cifar_test.cc | 11 ++- .../dataset/c_api_dataset_manifest_test.cc | 19 ++-- tests/ut/cpp/dataset/c_api_datasets_test.cc | 6 +- .../data/dataset/testManifestData/cpp2.json | 6 ++ 15 files changed, 200 insertions(+), 30 deletions(-) create mode 100644 tests/ut/data/dataset/testManifestData/cpp2.json diff --git a/mindspore/ccsrc/minddata/dataset/api/datasets.cc b/mindspore/ccsrc/minddata/dataset/api/datasets.cc index 8e245662ec..2c6ad6171a 100644 --- a/mindspore/ccsrc/minddata/dataset/api/datasets.cc +++ b/mindspore/ccsrc/minddata/dataset/api/datasets.cc @@ -257,32 +257,79 @@ int64_t Dataset::GetDatasetSize() { std::vector Dataset::GetOutputTypes() { std::vector types; - Status s; + Status rc; + std::unique_ptr runtime_context = std::make_unique(); + rc = runtime_context->Init(); + if (rc.IsError()) { + MS_LOG(ERROR) << "GetOutputTypes: Initializing RuntimeContext failed."; + types.clear(); + return types; + } if (!tree_getters_->isInitialized()) { - s = tree_getters_->Init(shared_from_this()); - if (s.IsError()) { - MS_LOG(ERROR) << "GetDatasetSize: Initializing RuntimeContext failed."; + rc = tree_getters_->Init(shared_from_this()); + if (rc.IsError()) { + MS_LOG(ERROR) << "GetOutputTypes: Initializing TreeGetters failed."; + types.clear(); return types; } } - tree_getters_->GetOutputTypes(&types); + rc = tree_getters_->GetOutputTypes(&types); + if (rc.IsError()) { + MS_LOG(ERROR) << "GetOutputTypes: Get Output Types failed."; + types.clear(); + return types; + } return types; } std::vector Dataset::GetOutputShapes() { std::vector shapes; - Status s; + Status rc; + std::unique_ptr runtime_context = std::make_unique(); + rc = runtime_context->Init(); + if (rc.IsError()) { + MS_LOG(ERROR) << "GetOutputShapes: Initializing RuntimeContext failed."; + shapes.clear(); + return shapes; + } if (!tree_getters_->isInitialized()) { - s = tree_getters_->Init(shared_from_this()); - if (s.IsError()) { - MS_LOG(ERROR) << "GetDatasetSize: Initializing RuntimeContext failed."; + rc = tree_getters_->Init(shared_from_this()); + if (rc.IsError()) { + MS_LOG(ERROR) << "GetOutputShapes: Initializing TreeGetters failed."; + shapes.clear(); return shapes; } } - tree_getters_->GetOutputShapes(&shapes); + rc = tree_getters_->GetOutputShapes(&shapes); + if (rc.IsError()) { + MS_LOG(ERROR) << "GetOutputShapes: Get Output Shapes failed."; + shapes.clear(); + return shapes; + } return shapes; } +int64_t Dataset::GetNumClasses() { + int64_t num_classes; + auto ds = shared_from_this(); + Status rc; + std::unique_ptr runtime_context = std::make_unique(); + rc = runtime_context->Init(); + if (rc.IsError()) { + MS_LOG(ERROR) << "GetNumClasses: Initializing RuntimeContext failed."; + return -1; + } + if (!tree_getters_->isInitialized()) { + rc = tree_getters_->Init(ds); + if (rc.IsError()) { + MS_LOG(ERROR) << "GetNumClasses: Initializing TreeGetters failed."; + return -1; + } + } + rc = tree_getters_->GetNumClasses(&num_classes); + return rc.IsError() ? -1 : num_classes; +} + // Constructor to initialize the cache Dataset::Dataset(const std::shared_ptr &dataset_cache) : Dataset() { cache_ = dataset_cache; } @@ -656,6 +703,7 @@ Status Dataset::AddCacheOp(std::vector> *node_ops) { } return Status::OK(); } + int64_t Dataset::GetBatchSize() { int64_t batch_size; auto ds = shared_from_this(); @@ -666,14 +714,17 @@ int64_t Dataset::GetBatchSize() { MS_LOG(ERROR) << "GetBatchSize: Initializing RuntimeContext failed."; return -1; } - rc = tree_getters_->Init(ds); - if (rc.IsError()) { - MS_LOG(ERROR) << "GetBatchSize: Initializing TreeGetters failed."; - return -1; + if (!tree_getters_->isInitialized()) { + rc = tree_getters_->Init(ds); + if (rc.IsError()) { + MS_LOG(ERROR) << "GetBatchSize: Initializing TreeGetters failed."; + return -1; + } } rc = tree_getters_->GetBatchSize(&batch_size); return rc.IsError() ? -1 : batch_size; } + int64_t Dataset::GetRepeatCount() { int64_t repeat_count; auto ds = shared_from_this(); @@ -684,10 +735,12 @@ int64_t Dataset::GetRepeatCount() { MS_LOG(ERROR) << "GetRepeatCount: Initializing RuntimeContext failed."; return -1; } - rc = tree_getters_->Init(ds); - if (rc.IsError()) { - MS_LOG(ERROR) << "GetRepeatCount: Initializing TreeGetters failed."; - return -1; + if (!tree_getters_->isInitialized()) { + rc = tree_getters_->Init(ds); + if (rc.IsError()) { + MS_LOG(ERROR) << "GetRepeatCount: Initializing TreeGetters failed."; + return -1; + } } rc = tree_getters_->GetRepeatCount(&repeat_count); return rc.IsError() ? 0 : repeat_count; diff --git a/mindspore/ccsrc/minddata/dataset/engine/consumers/tree_consumer.cc b/mindspore/ccsrc/minddata/dataset/engine/consumers/tree_consumer.cc index 53910bdcce..91a44663d8 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/consumers/tree_consumer.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/consumers/tree_consumer.cc @@ -438,10 +438,18 @@ Status TreeGetters::GetBatchSize(int64_t *batch_size) { CHECK_FAIL_RETURN_UNEXPECTED(*batch_size != -1, "Error in finding the batch size."); return Status::OK(); } + Status TreeGetters::GetRepeatCount(int64_t *repeat_count) { std::shared_ptr root = std::shared_ptr(tree_adapter_->GetRoot()); CHECK_FAIL_RETURN_UNEXPECTED(root != nullptr, "Root is a nullptr."); *repeat_count = root->GetTreeRepeatCount(); return Status::OK(); } + +Status TreeGetters::GetNumClasses(int64_t *num_classes) { + std::shared_ptr root = std::shared_ptr(tree_adapter_->GetRoot()); + CHECK_FAIL_RETURN_UNEXPECTED(root != nullptr, "Root is a nullptr."); + RETURN_IF_NOT_OK(root->GetNumClasses(num_classes)); + return Status::OK(); +} } // namespace mindspore::dataset diff --git a/mindspore/ccsrc/minddata/dataset/engine/consumers/tree_consumer.h b/mindspore/ccsrc/minddata/dataset/engine/consumers/tree_consumer.h index c7616f0034..1dcd314f9f 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/consumers/tree_consumer.h +++ b/mindspore/ccsrc/minddata/dataset/engine/consumers/tree_consumer.h @@ -164,6 +164,7 @@ class TreeGetters : public TreeConsumer { Status GetOutputShapes(std::vector *shapes); Status GetBatchSize(int64_t *batch_size); Status GetRepeatCount(int64_t *repeat_count); + Status GetNumClasses(int64_t *num_classes); bool isInitialized(); std::string Name() override { return "TreeGetters"; } Status GetRow(TensorRow *r); diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.cc index 8181e17896..e4f759c240 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.cc @@ -51,7 +51,8 @@ DatasetOp::DatasetOp(int32_t op_connector_size, std::shared_ptr sampler op_current_repeats_(0), op_current_epochs_(0), out_connector_(nullptr), - dataset_size_(-1) { + dataset_size_(-1), + num_classes_(-1) { // The operator starts out with an invalid operator id. The only way to // get it out of invalid state is to assign the operator to an execution tree. } @@ -302,6 +303,19 @@ Status DatasetOp::GetDatasetSize(int64_t *dataset_size) { return child_[0]->GetDatasetSize(dataset_size); } +// Gets the number of classes +Status DatasetOp::GetNumClasses(int64_t *num_classes) { + if (num_classes_ > 0) { + *num_classes = num_classes_; + return Status::OK(); + } + if (!child_.empty()) { + return child_[0]->GetNumClasses(num_classes); + } else { + RETURN_STATUS_UNEXPECTED("Can't get the dataset size for the current tree."); + } +} + // Performs handling for when an eoe message is received. // The base class implementation simply flows the eoe message to output. Derived classes // may override if they need to perform special eoe handling. diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.h b/mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.h index c8204e8426..1eebb8f08f 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.h +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/dataset_op.h @@ -191,6 +191,10 @@ class DatasetOp : public std::enable_shared_from_this { /// \return Status - The status code return virtual int64_t GetTreeRepeatCount(); + /// \brief Gets the number of classes + /// \return Status - The status code return + virtual Status GetNumClasses(int64_t *num_classes); + /// \brief Performs handling for when an eoe message is received. /// The base class implementation simply flows the eoe message to output. Derived classes /// may override if they need to perform special eoe handling. @@ -419,6 +423,7 @@ class DatasetOp : public std::enable_shared_from_this { std::mutex column_name_map_mutex_; // For protecting shared access to the column map CallbackManager callback_manager_; // Manages callbacks associated with a DatasetOp int64_t dataset_size_; // Size of the dataset + int64_t num_classes_; // Number of classes private: /// Sets the operator id. diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/image_folder_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/image_folder_op.cc index c08966d939..7a9d810eb0 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/image_folder_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/image_folder_op.cc @@ -468,5 +468,17 @@ Status ImageFolderOp::GetDatasetSize(int64_t *dataset_size) { dataset_size_ = *dataset_size; return Status::OK(); } + +// Get number of classes +Status ImageFolderOp::GetNumClasses(int64_t *num_classes) { + if (num_classes_ > 0) { + *num_classes = num_classes_; + return Status::OK(); + } + int64_t num_rows = num_rows_; + RETURN_IF_NOT_OK(CountRowsAndClasses(folder_path_, extensions_, &num_rows, num_classes)); + num_classes_ = *num_classes; + return Status::OK(); +} } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/image_folder_op.h b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/image_folder_op.h index 90b813f60b..4951039096 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/image_folder_op.h +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/image_folder_op.h @@ -218,6 +218,11 @@ class ImageFolderOp : public ParallelOp, public RandomAccessOp { /// \return Status of the function Status GetDatasetSize(int64_t *dataset_size) override; + /// \brief Base-class override for GetNumClasses + /// \param[out] num_classes the number of classes + /// \return Status of the function + Status GetNumClasses(int64_t *num_classes) override; + private: // Initialize Sampler, calls sampler->Init() within // @return Status - The error code return diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/manifest_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/manifest_op.cc index bfff2d4704..e0d08f93bb 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/manifest_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/manifest_op.cc @@ -18,6 +18,7 @@ #include #include #include +#include #include #include "utils/ms_utils.h" @@ -297,6 +298,7 @@ Status ManifestOp::ParseManifestFile() { RETURN_STATUS_UNEXPECTED("Invalid file, failed to open Manifest file: " + file_); } std::string line; + std::set classes; while (getline(file_handle, line)) { try { nlohmann::json js = nlohmann::json::parse(line); @@ -317,6 +319,7 @@ Status ManifestOp::ParseManifestFile() { for (nlohmann::json::iterator it = annotations.begin(); it != annotations.end(); ++it) { nlohmann::json annotation = it.value(); std::string label_name = annotation.value("name", ""); + classes.insert(label_name); if (label_name == "") { file_handle.close(); RETURN_STATUS_UNEXPECTED("Invalid data, label name is not found in Manifest file: " + image_file_path); @@ -336,6 +339,7 @@ Status ManifestOp::ParseManifestFile() { RETURN_STATUS_UNEXPECTED("Invalid file, failed to parse manifest file: " + line); } } + num_classes_ = classes.size(); file_handle.close(); return Status::OK(); @@ -471,5 +475,18 @@ Status ManifestOp::GetDatasetSize(int64_t *dataset_size) { return Status::OK(); } +// Get number of classes +Status ManifestOp::GetNumClasses(int64_t *num_classes) { + if (num_classes_ > 0) { + *num_classes = num_classes_; + return Status::OK(); + } + std::shared_ptr op; + RETURN_IF_NOT_OK(Builder().SetManifestFile(file_).SetClassIndex(class_index_).SetUsage(usage_).Build(&op)); + RETURN_IF_NOT_OK(op->ParseManifestFile()); + *num_classes = num_classes_; + return Status::OK(); +} + } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/manifest_op.h b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/manifest_op.h index ce19c06465..fa98b9c2a7 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/manifest_op.h +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/manifest_op.h @@ -188,6 +188,11 @@ class ManifestOp : public ParallelOp, public RandomAccessOp { /// \return Status of the function Status GetDatasetSize(int64_t *dataset_size) override; + /// \brief Base-class override for GetNumClasses + /// \param[out] num_classes the number of classes + /// \return Status of the function + Status GetNumClasses(int64_t *num_classes) override; + private: // Initialize Sampler, calls sampler->Init() within // @return Status - The error code return diff --git a/mindspore/ccsrc/minddata/dataset/include/datasets.h b/mindspore/ccsrc/minddata/dataset/include/datasets.h index 8a39ac7b4c..8149b6dfc2 100644 --- a/mindspore/ccsrc/minddata/dataset/include/datasets.h +++ b/mindspore/ccsrc/minddata/dataset/include/datasets.h @@ -589,15 +589,15 @@ class Dataset : public std::enable_shared_from_this { } /// \brief Gets the dataset size - /// \return int64_t + /// \return dataset size. If failed, return -1 int64_t GetDatasetSize(); /// \brief Gets the output type - /// \return vector of DataType + /// \return a vector of DataType. If failed, return an empty vector std::vector GetOutputTypes(); /// \brief Gets the output shape - /// \return vector of TensorShapes + /// \return a vector of TensorShape. If failed, return am empty vector std::vector GetOutputShapes(); /// \brief Gets the batch size @@ -608,6 +608,10 @@ class Dataset : public std::enable_shared_from_this { /// \return int64_t int64_t GetRepeatCount(); + /// \brief Gets the number of classes + /// \return number of classes. If failed, return -1 + int64_t GetNumClasses(); + /// \brief Setter function for runtime number of workers /// \param[in] num_workers The number of threads in this operator /// \return Shared pointer to the original object diff --git a/tests/ut/cpp/dataset/c_api_dataset_album_test.cc b/tests/ut/cpp/dataset/c_api_dataset_album_test.cc index c6c0ee663d..631d194a03 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_album_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_album_test.cc @@ -69,6 +69,26 @@ TEST_F(MindDataTestPipeline, TestAlbumBasic) { iter->Stop(); } +TEST_F(MindDataTestPipeline, TestAlbumgetters) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAlbumgetters."; + + std::string folder_path = datasets_root_path_ + "/testAlbum/images"; + std::string schema_file = datasets_root_path_ + "/testAlbum/datasetSchema.json"; + std::vector column_names = {"image", "label", "id"}; + // Create a Album Dataset + std::shared_ptr ds = Album(folder_path, schema_file, column_names); + EXPECT_NE(ds, nullptr); + + int64_t dataset_size = ds->GetDatasetSize(); + EXPECT_EQ(dataset_size, 7); + int64_t num_classes = ds->GetNumClasses(); + EXPECT_EQ(num_classes, -1); + int64_t batch_size = ds->GetBatchSize(); + EXPECT_EQ(batch_size, 1); + int64_t repeat_count = ds->GetRepeatCount(); + EXPECT_EQ(repeat_count, 1); +} + TEST_F(MindDataTestPipeline, TestAlbumDecode) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAlbumDecode."; std::string folder_path = datasets_root_path_ + "/testAlbum/images"; diff --git a/tests/ut/cpp/dataset/c_api_dataset_cifar_test.cc b/tests/ut/cpp/dataset/c_api_dataset_cifar_test.cc index f081764734..0b91c9af5e 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_cifar_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_cifar_test.cc @@ -86,7 +86,7 @@ TEST_F(MindDataTestPipeline, TestCifar10GetDatasetSize) { EXPECT_EQ(ds->GetDatasetSize(), 10000); } -TEST_F(MindDataTestPipeline, TestCifar10MixGetter) { +TEST_F(MindDataTestPipeline, TestCifar10Getters) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestCifar10MixGetter."; // Create a Cifar10 Dataset @@ -97,19 +97,28 @@ TEST_F(MindDataTestPipeline, TestCifar10MixGetter) { EXPECT_EQ(ds->GetDatasetSize(), 10000); std::vector types = ds->GetOutputTypes(); std::vector shapes = ds->GetOutputShapes(); + int64_t num_classes = ds->GetNumClasses(); EXPECT_EQ(types.size(), 2); EXPECT_EQ(types[0].ToString(), "uint8"); EXPECT_EQ(types[1].ToString(), "uint32"); EXPECT_EQ(shapes.size(), 2); EXPECT_EQ(shapes[0].ToString(), "<32,32,3>"); EXPECT_EQ(shapes[1].ToString(), "<>"); + EXPECT_EQ(num_classes, -1); + EXPECT_EQ(ds->GetBatchSize(), 1); + EXPECT_EQ(ds->GetRepeatCount(), 1); EXPECT_EQ(ds->GetDatasetSize(), 10000); EXPECT_EQ(ds->GetOutputTypes(), types); EXPECT_EQ(ds->GetOutputShapes(), shapes); + EXPECT_EQ(ds->GetNumClasses(), -1); + EXPECT_EQ(ds->GetDatasetSize(), 10000); EXPECT_EQ(ds->GetOutputTypes(), types); EXPECT_EQ(ds->GetOutputShapes(), shapes); + EXPECT_EQ(ds->GetBatchSize(), 1); + EXPECT_EQ(ds->GetRepeatCount(), 1); + EXPECT_EQ(ds->GetNumClasses(), -1); EXPECT_EQ(ds->GetDatasetSize(), 10000); } diff --git a/tests/ut/cpp/dataset/c_api_dataset_manifest_test.cc b/tests/ut/cpp/dataset/c_api_dataset_manifest_test.cc index f36a8848ea..9ef0398dba 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_manifest_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_manifest_test.cc @@ -67,15 +67,22 @@ TEST_F(MindDataTestPipeline, TestManifestBasic) { iter->Stop(); } -TEST_F(MindDataTestPipeline, TestManifestGetDatasetSize) { - MS_LOG(INFO) << "Doing MindDataTestPipeline-TestManifestGetDatasetSize."; +TEST_F(MindDataTestPipeline, TestManifestGetters) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestManifestGetters."; - std::string file_path = datasets_root_path_ + "/testManifestData/cpp.json"; + std::string file_path1 = datasets_root_path_ + "/testManifestData/cpp.json"; + std::string file_path2 = datasets_root_path_ + "/testManifestData/cpp2.json"; // Create a Manifest Dataset - std::shared_ptr ds = Manifest(file_path); - EXPECT_NE(ds, nullptr); + std::shared_ptr ds1 = Manifest(file_path1); + std::shared_ptr ds2 = Manifest(file_path2); + + EXPECT_NE(ds1, nullptr); + EXPECT_EQ(ds1->GetDatasetSize(), 2); + EXPECT_EQ(ds1->GetNumClasses(), 2); - EXPECT_EQ(ds->GetDatasetSize(), 2); + EXPECT_NE(ds2, nullptr); + EXPECT_EQ(ds2->GetDatasetSize(), 4); + EXPECT_EQ(ds2->GetNumClasses(), 3); } TEST_F(MindDataTestPipeline, TestManifestDecode) { diff --git a/tests/ut/cpp/dataset/c_api_datasets_test.cc b/tests/ut/cpp/dataset/c_api_datasets_test.cc index ab1c08d6ba..6c4660769b 100644 --- a/tests/ut/cpp/dataset/c_api_datasets_test.cc +++ b/tests/ut/cpp/dataset/c_api_datasets_test.cc @@ -221,7 +221,7 @@ TEST_F(MindDataTestPipeline, TestImageFolderFailWithWrongExtension) { iter->Stop(); } -TEST_F(MindDataTestPipeline, TestImageFolderGetDatasetSize) { +TEST_F(MindDataTestPipeline, TestImageFolderGetters) { MS_LOG(INFO) << "Doing MindDataTestPipeline-TestImageFolderGetDatasetSize."; // Create an ImageFolder Dataset @@ -230,6 +230,10 @@ TEST_F(MindDataTestPipeline, TestImageFolderGetDatasetSize) { EXPECT_NE(ds, nullptr); EXPECT_EQ(ds->GetDatasetSize(), 44); + EXPECT_EQ(ds->GetNumClasses(), 4); + EXPECT_EQ(ds->GetNumClasses(), 4); + EXPECT_EQ(ds->GetDatasetSize(), 44); + EXPECT_EQ(ds->GetDatasetSize(), 44); } TEST_F(MindDataTestPipeline, TestImageFolderFailWithNullSampler) { diff --git a/tests/ut/data/dataset/testManifestData/cpp2.json b/tests/ut/data/dataset/testManifestData/cpp2.json new file mode 100644 index 0000000000..a318b92549 --- /dev/null +++ b/tests/ut/data/dataset/testManifestData/cpp2.json @@ -0,0 +1,6 @@ +{"source":"./data/dataset/testManifestData/train/1.JPEG", "usage":"TRAIN","id":"0162005993f8065ef47eefb59d1e4970","annotation": [{"type": "modelarts/image_classification","name": "dog","property": {"color":"white","kind":"Persian cat"},"hard":"true","hard-coefficient":0.8,"annotated-by":"human","creation-time":"2019-01-23 11:30:30"}],"inference-loc":"/path/to/inference-output"} +{"source":"./data/dataset/testManifestData/train/1.JPEG", "usage":"TRAIN","id":"0162005993f8065ef47eefb59d1e4970","annotation": [{"type": "modelarts/image_classification","name": "cat","property": {"color":"white","kind":"Persian cat"},"hard":"true","hard-coefficient":0.8,"annotated-by":"human","creation-time":"2019-01-23 11:30:30"}],"inference-loc":"/path/to/inference-output"} +{"source":"./data/dataset/testManifestData/train/1.JPEG", "usage":"TRAIN","id":"0162005993f8065ef47eefb59d1e4970","annotation": [{"type": "modelarts/image_classification","name": "cat","property": {"color":"white","kind":"Persian cat"},"hard":"true","hard-coefficient":0.8,"annotated-by":"human","creation-time":"2019-01-23 11:30:30"}],"inference-loc":"/path/to/inference-output"} +{"source":"./data/dataset/testManifestData/train/1.JPEG", "usage":"TRAIN","id":"0162005993f8065ef47eefb59d1e4970","annotation": [{"type": "modelarts/image_classification","name": "cat","property": {"color":"white","kind":"Persian cat"},"hard":"true","hard-coefficient":0.8,"annotated-by":"human","creation-time":"2019-01-23 11:30:30"},{"type": "modelarts/image_classification","name": "flower","property": {"color":"white","kind":"Persian cat"},"hard":"true","hard-coefficient":0.8,"annotated-by":"human","creation-time":"2019-01-23 11:30:30"}],"inference-loc":"/path/to/inference-output"} +{"source":"./data/dataset/testManifestData/eval/1.JPEG", "usage":"EVAL","id":"0162005993f8065ef47eefb59d1e4970","annotation": [{"type": "modelarts/image_classification","name": "cat","property": {"color":"white","kind":"Persian cat"},"hard":"true","hard-coefficient":0.8,"annotated-by":"human","creation-time":"2019-01-23 11:30:30"}],"inference-loc":"/path/to/inference-output"} +{"source":"./data/dataset/testManifestData/eval/2.JPEG", "usage":"EVAL","id":"0162005993f8065ef47eefb59d1e4970","annotation": [{"type": "modelarts/image_classification","name": "dog","property": {"color":"white","kind":"Persian cat"},"hard":"true","hard-coefficient":0.8,"annotated-by":"human","creation-time":"2019-01-23 11:30:30"}],"inference-loc":"/path/to/inference-output"}