From 521c313a3b79cf34cc9a7e7162d37f03060487b8 Mon Sep 17 00:00:00 2001 From: Cathy Wong Date: Tue, 2 Mar 2021 16:55:50 -0500 Subject: [PATCH] dataset: Reinstate INFO logging and data verification - part1 --- .../cpp/dataset/c_api_dataset_album_test.cc | 22 +- .../cpp/dataset/c_api_dataset_cifar_test.cc | 12 +- .../ut/cpp/dataset/c_api_dataset_clue_test.cc | 43 ++-- .../ut/cpp/dataset/c_api_dataset_coco_test.cc | 24 +- .../ut/cpp/dataset/c_api_dataset_csv_test.cc | 2 +- .../dataset/c_api_dataset_iterator_test.cc | 58 +++-- .../dataset/c_api_dataset_manifest_test.cc | 30 +-- .../dataset/c_api_dataset_minddata_test.cc | 16 +- .../ut/cpp/dataset/c_api_dataset_ops_test.cc | 78 +++---- .../dataset/c_api_dataset_randomdata_test.cc | 213 ++++++++++-------- .../dataset/c_api_dataset_textfile_test.cc | 46 ++-- .../dataset/c_api_dataset_tfrecord_test.cc | 51 +++-- .../ut/cpp/dataset/c_api_dataset_voc_test.cc | 24 +- tests/ut/cpp/dataset/c_api_datasets_test.cc | 8 +- tests/ut/cpp/dataset/ir_vision_test.cc | 2 +- 15 files changed, 321 insertions(+), 308 deletions(-) diff --git a/tests/ut/cpp/dataset/c_api_dataset_album_test.cc b/tests/ut/cpp/dataset/c_api_dataset_album_test.cc index c1488b1e40..d607546cb3 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_album_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_album_test.cc @@ -45,8 +45,8 @@ TEST_F(MindDataTestPipeline, TestAlbumBasic) { uint64_t i = 0; while (row.size() != 0) { i++; - // auto image = row["image"]; - // MS_LOG(INFO) << "Tensor image shape: " << image->shape(); + auto image = row["image"]; + MS_LOG(INFO) << "Tensor image shape: " << image.Shape(); iter->GetNextRow(&row); } @@ -100,8 +100,8 @@ TEST_F(MindDataTestPipeline, TestAlbumBasicWithPipeline) { uint64_t i = 0; while (row.size() != 0) { i++; - // auto image = row["image"]; - // MS_LOG(INFO) << "Tensor image shape: " << image->shape(); + auto image = row["image"]; + MS_LOG(INFO) << "Tensor image shape: " << image.Shape(); iter->GetNextRow(&row); } @@ -151,13 +151,11 @@ TEST_F(MindDataTestPipeline, TestAlbumDecode) { uint64_t i = 0; while (row.size() != 0) { i++; - /* auto image = row["image"]; - auto shape = image->shape(); - MS_LOG(INFO) << "Tensor image shape size: " << shape.Size(); - MS_LOG(INFO) << "Tensor image shape: " << image->shape(); - EXPECT_GT(shape.Size(), 1); // Verify decode=true took effect - */ + auto shape = image.Shape(); + MS_LOG(INFO) << "Tensor image shape size: " << shape.size(); + MS_LOG(INFO) << "Tensor image shape: " << image.Shape(); + EXPECT_GT(shape.size(), 1); // Verify decode=true took effect iter->GetNextRow(&row); } @@ -189,8 +187,8 @@ TEST_F(MindDataTestPipeline, TestAlbumNumSamplers) { uint64_t i = 0; while (row.size() != 0) { i++; - // auto image = row["image"]; - // MS_LOG(INFO) << "Tensor image shape: " << image->shape(); + auto image = row["image"]; + MS_LOG(INFO) << "Tensor image shape: " << image.Shape(); iter->GetNextRow(&row); } diff --git a/tests/ut/cpp/dataset/c_api_dataset_cifar_test.cc b/tests/ut/cpp/dataset/c_api_dataset_cifar_test.cc index e2fc4243ea..a8fd2daffb 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_cifar_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_cifar_test.cc @@ -48,8 +48,8 @@ TEST_F(MindDataTestPipeline, TestCifar10Dataset) { uint64_t i = 0; while (row.size() != 0) { i++; - // auto image = row["image"]; - // MS_LOG(INFO) << "Tensor image shape: " << image->shape(); + auto image = row["image"]; + MS_LOG(INFO) << "Tensor image shape: " << image.Shape(); iter->GetNextRow(&row); } @@ -103,8 +103,8 @@ TEST_F(MindDataTestPipeline, TestCifar10DatasetWithPipeline) { uint64_t i = 0; while (row.size() != 0) { i++; - // auto image = row["image"]; - // MS_LOG(INFO) << "Tensor image shape: " << image->shape(); + auto image = row["image"]; + MS_LOG(INFO) << "Tensor image shape: " << image.Shape(); iter->GetNextRow(&row); } @@ -187,8 +187,8 @@ TEST_F(MindDataTestPipeline, TestCifar100Dataset) { uint64_t i = 0; while (row.size() != 0) { i++; - // auto image = row["image"]; - // MS_LOG(INFO) << "Tensor image shape: " << image->shape(); + auto image = row["image"]; + MS_LOG(INFO) << "Tensor image shape: " << image.Shape(); iter->GetNextRow(&row); } diff --git a/tests/ut/cpp/dataset/c_api_dataset_clue_test.cc b/tests/ut/cpp/dataset/c_api_dataset_clue_test.cc index ace2becf19..30dda9633d 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_clue_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_clue_test.cc @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -53,12 +53,12 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetAFQMC) { uint64_t i = 0; while (row.size() != 0) { - // auto text = row["sentence1"]; + auto text = row["sentence1"]; // std::string_view sv; // text->GetItemAt(&sv, {0}); // std::string ss(sv); // EXPECT_STREQ(ss.c_str(), expected_result[i].c_str()); - // MS_LOG(INFO) << "Tensor text shape: " << text->shape(); + MS_LOG(INFO) << "Tensor text shape: " << text.Shape(); iter->GetNextRow(&row); i++; } @@ -134,8 +134,8 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetBasic) { EXPECT_NE(row.find("sentence1"), row.end()); uint64_t i = 0; while (row.size() != 0) { - // auto text = row["sentence1"]; - // MS_LOG(INFO) << "Tensor text shape: " << text->shape(); + auto text = row["sentence1"]; + MS_LOG(INFO) << "Tensor text shape: " << text.Shape(); i++; iter->GetNextRow(&row); } @@ -190,8 +190,8 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetBasicWithPipeline) { EXPECT_NE(row.find("sentence1"), row.end()); uint64_t i = 0; while (row.size() != 0) { - // auto text = row["sentence1"]; - // MS_LOG(INFO) << "Tensor text shape: " << text->shape(); + auto text = row["sentence1"]; + MS_LOG(INFO) << "Tensor text shape: " << text.Shape(); i++; iter->GetNextRow(&row); } @@ -242,12 +242,12 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetCMNLI) { uint64_t i = 0; while (row.size() != 0) { - // auto text = row["sentence1"]; + auto text = row["sentence1"]; // std::string_view sv; // text->GetItemAt(&sv, {0}); // std::string ss(sv); // EXPECT_STREQ(ss.c_str(), expected_result[i].c_str()); - // MS_LOG(INFO) << "Tensor text shape: " << text->shape(); + MS_LOG(INFO) << "Tensor text shape: " << text.Shape(); iter->GetNextRow(&row); i++; } @@ -283,12 +283,12 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetCSL) { uint64_t i = 0; while (row.size() != 0) { - // auto text = row["abst"]; + auto text = row["abst"]; // std::string_view sv; // text->GetItemAt(&sv, {0}); // std::string ss(sv); // EXPECT_STREQ(ss.c_str(), expected_result[i].c_str()); - // MS_LOG(INFO) << "Tensor text shape: " << text->shape(); + MS_LOG(INFO) << "Tensor text shape: " << text.Shape(); iter->GetNextRow(&row); i++; } @@ -322,8 +322,8 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetDistribution) { EXPECT_NE(row.find("sentence1"), row.end()); uint64_t i = 0; while (row.size() != 0) { - // auto text = row["sentence1"]; - // MS_LOG(INFO) << "Tensor text shape: " << text->shape(); + auto text = row["sentence1"]; + MS_LOG(INFO) << "Tensor text shape: " << text.Shape(); i++; iter->GetNextRow(&row); } @@ -424,12 +424,12 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetIFLYTEK) { uint64_t i = 0; while (row.size() != 0) { - // auto text = row["sentence"]; + auto text = row["sentence"]; // std::string_view sv; // text->GetItemAt(&sv, {0}); // std::string ss(sv); // EXPECT_STREQ(ss.c_str(), expected_result[i].c_str()); - // MS_LOG(INFO) << "Tensor text shape: " << text->shape(); + MS_LOG(INFO) << "Tensor text shape: " << text.Shape(); iter->GetNextRow(&row); i++; } @@ -602,13 +602,12 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetShuffleGlobal) { // "蚂蚁借呗等额还款能否换成先息后本"}; uint64_t i = 0; while (row.size() != 0) { - // auto text = row["sentence1"]; - // MS_LOG(INFO) << "Tensor text shape: " << text->shape(); + auto text = row["sentence1"]; // std::string_view sv; // text->GetItemAt(&sv, {0}); // std::string ss(sv); // EXPECT_STREQ(ss.c_str(), expected_result[i].c_str()); - // MS_LOG(INFO) << "Tensor text shape: " << text->shape(); + MS_LOG(INFO) << "Tensor text shape: " << text.Shape(); i++; iter->GetNextRow(&row); } @@ -648,12 +647,12 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetTNEWS) { uint64_t i = 0; while (row.size() != 0) { - // auto text = row["sentence"]; + auto text = row["sentence"]; // std::string_view sv; // text->GetItemAt(&sv, {0}); // std::string ss(sv); // EXPECT_STREQ(ss.c_str(), expected_result[i].c_str()); - // MS_LOG(INFO) << "Tensor text shape: " << text->shape(); + MS_LOG(INFO) << "Tensor text shape: " << text.Shape(); iter->GetNextRow(&row); i++; } @@ -690,12 +689,12 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetWSC) { uint64_t i = 0; while (row.size() != 0) { - // auto text = row["text"]; + auto text = row["text"]; // std::string_view sv; // text->GetItemAt(&sv, {0}); // std::string ss(sv); // EXPECT_STREQ(ss.c_str(), expected_result[i].c_str()); - // MS_LOG(INFO) << "Tensor text shape: " << text->shape(); + MS_LOG(INFO) << "Tensor text shape: " << text.Shape(); iter->GetNextRow(&row); i++; } diff --git a/tests/ut/cpp/dataset/c_api_dataset_coco_test.cc b/tests/ut/cpp/dataset/c_api_dataset_coco_test.cc index 6f5475b47d..5e394a0a52 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_coco_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_coco_test.cc @@ -45,12 +45,12 @@ TEST_F(MindDataTestPipeline, TestCocoDefault) { uint64_t i = 0; while (row.size() != 0) { - // auto image = row["image"]; - // auto bbox = row["bbox"]; - // auto category_id = row["category_id"]; - // MS_LOG(INFO) << "Tensor image shape: " << image->shape(); - // MS_LOG(INFO) << "Tensor bbox shape: " << bbox->shape(); - // MS_LOG(INFO) << "Tensor category_id shape: " << category_id->shape(); + auto image = row["image"]; + auto bbox = row["bbox"]; + auto category_id = row["category_id"]; + MS_LOG(INFO) << "Tensor image shape: " << image.Shape(); + MS_LOG(INFO) << "Tensor bbox shape: " << bbox.Shape(); + MS_LOG(INFO) << "Tensor category_id shape: " << category_id.Shape(); iter->GetNextRow(&row); i++; } @@ -102,12 +102,12 @@ TEST_F(MindDataTestPipeline, TestCocoDefaultWithPipeline) { uint64_t i = 0; while (row.size() != 0) { - // auto image = row["image"]; - // auto bbox = row["bbox"]; - // auto category_id = row["category_id"]; - // MS_LOG(INFO) << "Tensor image shape: " << image->shape(); - // MS_LOG(INFO) << "Tensor bbox shape: " << bbox->shape(); - // MS_LOG(INFO) << "Tensor category_id shape: " << category_id->shape(); + auto image = row["image"]; + auto bbox = row["bbox"]; + auto category_id = row["category_id"]; + MS_LOG(INFO) << "Tensor image shape: " << image.Shape(); + MS_LOG(INFO) << "Tensor bbox shape: " << bbox.Shape(); + MS_LOG(INFO) << "Tensor category_id shape: " << category_id.Shape(); iter->GetNextRow(&row); i++; } diff --git a/tests/ut/cpp/dataset/c_api_dataset_csv_test.cc b/tests/ut/cpp/dataset/c_api_dataset_csv_test.cc index c4f3f3482b..12b5cf09a8 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_csv_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_csv_test.cc @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/tests/ut/cpp/dataset/c_api_dataset_iterator_test.cc b/tests/ut/cpp/dataset/c_api_dataset_iterator_test.cc index 5c69cb5005..73ec59aaa5 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_iterator_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_iterator_test.cc @@ -1,4 +1,3 @@ - /** * Copyright 2020-2021 Huawei Technologies Co., Ltd * @@ -18,8 +17,6 @@ #include "minddata/dataset/include/datasets.h" using namespace mindspore::dataset; -using mindspore::dataset::Tensor; -using mindspore::dataset::TensorShape; class MindDataTestPipeline : public UT::DatasetOpTesting { protected: @@ -43,14 +40,15 @@ TEST_F(MindDataTestPipeline, TestIteratorEmptyColumn) { // Iterate the dataset and get each row std::vector row; iter->GetNextRow(&row); - // TensorShape expect0({32, 32, 3}); - // TensorShape expect1({}); + std::vector expect_image = {32, 32, 3}; + std::vector expect_label = {}; uint64_t i = 0; while (row.size() != 0) { - // MS_LOG(INFO) << "row[0]:" << row[0]->shape() << ", row[1]:" << row[1]->shape(); - // EXPECT_EQ(expect0, row[0]->shape()); - // EXPECT_EQ(expect1, row[1]->shape()); + MS_LOG(INFO) << "row[0]:" << row[0].Shape() << ", row[1]:" << row[1].Shape(); + EXPECT_EQ(expect_image, row[0].Shape()); + EXPECT_EQ(expect_label, row[1].Shape()); + iter->GetNextRow(&row); i++; } @@ -82,14 +80,14 @@ TEST_F(MindDataTestPipeline, TestIteratorOneColumn) { // Iterate the dataset and get each row std::vector row; iter->GetNextRow(&row); - // TensorShape expect({2, 28, 28, 1}); + std::vector expect_image = {2, 28, 28, 1}; uint64_t i = 0; while (row.size() != 0) { - // for (auto &v : row) { - // MS_LOG(INFO) << "image shape:" << v->shape(); - // EXPECT_EQ(expect, v->shape()); - // } + for (auto &v : row) { + MS_LOG(INFO) << "image shape:" << v.Shape(); + EXPECT_EQ(expect_image, v.Shape()); + } iter->GetNextRow(&row); i++; } @@ -120,16 +118,15 @@ TEST_F(MindDataTestPipeline, TestIteratorReOrder) { // Iterate the dataset and get each row std::vector row; iter->GetNextRow(&row); - // TensorShape expect0({32, 32, 3}); - // TensorShape expect1({}); + std::vector expect_image = {32, 32, 3}; + std::vector expect_label = {}; - // Check if we will catch "label" before "image" in row - // std::vector expect = {"label", "image"}; + // Check "label" before "image" in row uint64_t i = 0; while (row.size() != 0) { - // MS_LOG(INFO) << "row[0]:" << row[0]->shape() << ", row[1]:" << row[1]->shape(); - // EXPECT_EQ(expect1, row[0]->shape()); - // EXPECT_EQ(expect0, row[1]->shape()); + MS_LOG(INFO) << "row[0]:" << row[0].Shape() << ", row[1]:" << row[1].Shape(); + EXPECT_EQ(expect_label, row[0].Shape()); + EXPECT_EQ(expect_image, row[1].Shape()); iter->GetNextRow(&row); i++; } @@ -161,20 +158,19 @@ TEST_F(MindDataTestPipeline, TestIteratorTwoColumns) { // Iterate the dataset and get each row std::vector row; iter->GetNextRow(&row); - // std::vector expect = {TensorShape({173673}), TensorShape({1, 4}), TensorShape({173673}), - // TensorShape({1, 4}), TensorShape({147025}), TensorShape({1, 4}), - // TensorShape({211653}), TensorShape({1, 4})}; + std::vector> expect = {{173673}, {1, 4}, {173673}, {1, 4}, + {147025}, {1, 4}, {211653}, {1, 4}}; uint64_t i = 0; - // uint64_t j = 0; + uint64_t j = 0; while (row.size() != 0) { - // MS_LOG(INFO) << "row[0]:" << row[0]->shape() << ", row[1]:" << row[1]->shape(); - // EXPECT_EQ(2, row.size()); - // EXPECT_EQ(expect[j++], row[0]->shape()); - // EXPECT_EQ(expect[j++], row[1]->shape()); + MS_LOG(INFO) << "row[0]:" << row[0].Shape() << ", row[1]:" << row[1].Shape(); + EXPECT_EQ(2, row.size()); + EXPECT_EQ(expect[j++], row[0].Shape()); + EXPECT_EQ(expect[j++], row[1].Shape()); iter->GetNextRow(&row); i++; - // j = (j == expect.size()) ? 0 : j; + j = (j == expect.size()) ? 0 : j; } EXPECT_EQ(i, 8); @@ -222,8 +218,8 @@ TEST_F(MindDataTestPipeline, TestIteratorNumEpoch) { EXPECT_EQ(inner_row_cnt, random_data_num_row); } EXPECT_EQ(total_row_cnt, random_data_num_row * num_epochs); - // this will go beyond the random_data_num_row*num_epoch limit, hence error code is expected - EXPECT_FALSE(iter->GetNextRow(&row)); + // This will go beyond the random_data_num_row*num_epoch limit, hence error code is expected + EXPECT_ERROR(iter->GetNextRow(&row)); // Manually terminate the pipeline iter->Stop(); } diff --git a/tests/ut/cpp/dataset/c_api_dataset_manifest_test.cc b/tests/ut/cpp/dataset/c_api_dataset_manifest_test.cc index a63a729080..c621878d03 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_manifest_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_manifest_test.cc @@ -43,8 +43,8 @@ TEST_F(MindDataTestPipeline, TestManifestBasic) { uint64_t i = 0; while (row.size() != 0) { i++; - // auto image = row["image"]; - // MS_LOG(INFO) << "Tensor image shape: " << image->shape(); + auto image = row["image"]; + MS_LOG(INFO) << "Tensor image shape: " << image.Shape(); iter->GetNextRow(&row); } @@ -95,8 +95,8 @@ TEST_F(MindDataTestPipeline, TestManifestBasicWithPipeline) { uint64_t i = 0; while (row.size() != 0) { i++; - // auto image = row["image"]; - // MS_LOG(INFO) << "Tensor image shape: " << image->shape(); + auto image = row["image"]; + MS_LOG(INFO) << "Tensor image shape: " << image.Shape(); iter->GetNextRow(&row); } @@ -162,11 +162,11 @@ TEST_F(MindDataTestPipeline, TestManifestDecode) { uint64_t i = 0; while (row.size() != 0) { i++; - // auto image = row["image"]; - // auto shape = image->shape(); - // MS_LOG(INFO) << "Tensor image shape size: " << shape.Size(); - // MS_LOG(INFO) << "Tensor image shape: " << image->shape(); - // EXPECT_GT(shape.Size(), 1); // Verify decode=true took effect + auto image = row["image"]; + auto shape = image.Shape(); + MS_LOG(INFO) << "Tensor image shape size: " << shape.size(); + MS_LOG(INFO) << "Tensor image shape: " << image.Shape(); + EXPECT_GT(shape.size(), 1); // Verify decode=true took effect iter->GetNextRow(&row); } @@ -196,8 +196,8 @@ TEST_F(MindDataTestPipeline, TestManifestEval) { uint64_t i = 0; while (row.size() != 0) { i++; - // auto image = row["image"]; - // MS_LOG(INFO) << "Tensor image shape: " << image->shape(); + auto image = row["image"]; + MS_LOG(INFO) << "Tensor image shape: " << image.Shape(); iter->GetNextRow(&row); } @@ -241,8 +241,8 @@ TEST_F(MindDataTestPipeline, TestManifestClassIndex) { // int32_t label_idx = 0; while (row.size() != 0) { i++; - // auto image = row["image"]; - // MS_LOG(INFO) << "Tensor image shape: " << image->shape(); + auto image = row["image"]; + MS_LOG(INFO) << "Tensor image shape: " << image.Shape(); // row["label"]->GetItemAt(&label_idx, {}); // MS_LOG(INFO) << "Tensor label value: " << label_idx; // auto label_it = std::find(expected_label.begin(), expected_label.end(), label_idx); @@ -276,8 +276,8 @@ TEST_F(MindDataTestPipeline, TestManifestNumSamplers) { uint64_t i = 0; while (row.size() != 0) { i++; - // auto image = row["image"]; - // MS_LOG(INFO) << "Tensor image shape: " << image->shape(); + auto image = row["image"]; + MS_LOG(INFO) << "Tensor image shape: " << image.Shape(); iter->GetNextRow(&row); } diff --git a/tests/ut/cpp/dataset/c_api_dataset_minddata_test.cc b/tests/ut/cpp/dataset/c_api_dataset_minddata_test.cc index d58301764a..c00a4a02e5 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_minddata_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_minddata_test.cc @@ -246,7 +246,7 @@ TEST_F(MindDataTestPipeline, TestMindDataSuccess6) { EXPECT_NE(ds5, nullptr); std::vector> ds = {ds1, ds2, ds3, ds4, ds5, ds6}; - // std::vector expected_samples = {5, 5, 2, 3, 3, 2}; + std::vector expected_samples = {5, 5, 2, 3, 3, 2}; for (int32_t i = 0; i < ds.size(); i++) { // Create an iterator over the result of the above dataset @@ -258,13 +258,13 @@ TEST_F(MindDataTestPipeline, TestMindDataSuccess6) { std::unordered_map row; iter->GetNextRow(&row); - // uint64_t j = 0; - // while (row.size() != 0) { - // j++; - // MS_LOG(INFO) << "Tensor label: " << *row["label"]; - // iter->GetNextRow(&row); - // } - // EXPECT_EQ(j, expected_samples[i]); + uint64_t j = 0; + while (row.size() != 0) { + j++; + // MS_LOG(INFO) << "Tensor label: " << *row["label"]; + iter->GetNextRow(&row); + } + EXPECT_EQ(j, expected_samples[i]); // Manually terminate the pipeline iter->Stop(); diff --git a/tests/ut/cpp/dataset/c_api_dataset_ops_test.cc b/tests/ut/cpp/dataset/c_api_dataset_ops_test.cc index dc0f2f9368..85e054ca47 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_ops_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_ops_test.cc @@ -101,8 +101,8 @@ TEST_F(MindDataTestPipeline, TestBatchAndRepeat) { uint64_t i = 0; while (row.size() != 0) { i++; - // auto image = row["image"]; - // MS_LOG(INFO) << "Tensor image shape: " << image->shape(); + auto image = row["image"]; + MS_LOG(INFO) << "Tensor image shape: " << image.Shape(); iter->GetNextRow(&row); } @@ -137,8 +137,8 @@ TEST_F(MindDataTestPipeline, TestBucketBatchByLengthSuccess1) { uint64_t i = 0; while (row.size() != 0) { i++; - // auto image = row["image"]; - // MS_LOG(INFO) << "Tensor image shape: " << image->shape(); + auto image = row["image"]; + MS_LOG(INFO) << "Tensor image shape: " << image.Shape(); iter->GetNextRow(&row); } // 2 batches of size 5 @@ -174,8 +174,8 @@ TEST_F(MindDataTestPipeline, TestBucketBatchByLengthSuccess2) { uint64_t i = 0; while (row.size() != 0) { i++; - // auto image = row["image"]; - // MS_LOG(INFO) << "Tensor image shape: " << image->shape(); + auto image = row["image"]; + MS_LOG(INFO) << "Tensor image shape: " << image.Shape(); iter->GetNextRow(&row); } // With 2 boundaries, 3 buckets are created @@ -486,8 +486,8 @@ TEST_F(MindDataTestPipeline, TestConcatSuccess) { uint64_t i = 0; while (row.size() != 0) { i++; - // auto image = row["image"]; - // MS_LOG(INFO) << "Tensor image shape: " << image->shape(); + auto image = row["image"]; + MS_LOG(INFO) << "Tensor image shape: " << image.Shape(); iter->GetNextRow(&row); } @@ -566,8 +566,8 @@ TEST_F(MindDataTestPipeline, TestConcatSuccess2) { uint64_t i = 0; while (row.size() != 0) { i++; - // auto image = row["image"]; - // MS_LOG(INFO) << "Tensor image shape: " << image->shape(); + auto image = row["image"]; + MS_LOG(INFO) << "Tensor image shape: " << image.Shape(); iter->GetNextRow(&row); } @@ -780,8 +780,8 @@ TEST_F(MindDataTestPipeline, TestImageFolderBatchAndRepeat) { uint64_t i = 0; while (row.size() != 0) { i++; - // auto image = row["image"]; - // MS_LOG(INFO) << "Tensor image shape: " << image->shape(); + auto image = row["image"]; + MS_LOG(INFO) << "Tensor image shape: " << image.Shape(); iter->GetNextRow(&row); } @@ -917,8 +917,8 @@ TEST_F(MindDataTestPipeline, TestProjectMap) { uint64_t i = 0; while (row.size() != 0) { i++; - // auto image = row["image"]; - // MS_LOG(INFO) << "Tensor image shape: " << image->shape(); + auto image = row["image"]; + MS_LOG(INFO) << "Tensor image shape: " << image.Shape(); iter->GetNextRow(&row); } @@ -1055,9 +1055,9 @@ TEST_F(MindDataTestPipeline, TestProjectMapAutoInjection) { uint64_t i = 0; while (row.size() != 0) { i++; - // auto image = row["image"]; - // MS_LOG(INFO) << "Tensor image shape: " << image->shape(); - // EXPECT_EQ(image->shape()[0], 30); + auto image = row["image"]; + MS_LOG(INFO) << "Tensor image shape: " << image.Shape(); + // EXPECT_EQ(image.Shape()[0], 30); iter->GetNextRow(&row); } @@ -1177,8 +1177,8 @@ TEST_F(MindDataTestPipeline, TestRenameSuccess) { while (row.size() != 0) { i++; - // auto image = row["col1"]; - // MS_LOG(INFO) << "Tensor image shape: " << image->shape(); + auto image = row["col1"]; + MS_LOG(INFO) << "Tensor image shape: " << image.Shape(); iter->GetNextRow(&row); } @@ -1221,8 +1221,8 @@ TEST_F(MindDataTestPipeline, TestRepeatDefault) { break; } i++; - // auto image = row["image"]; - // MS_LOG(INFO) << "Tensor image shape: " << image->shape(); + auto image = row["image"]; + MS_LOG(INFO) << "Tensor image shape: " << image.Shape(); iter->GetNextRow(&row); } @@ -1260,8 +1260,8 @@ TEST_F(MindDataTestPipeline, TestRepeatOne) { uint64_t i = 0; while (row.size() != 0) { i++; - // auto image = row["image"]; - // MS_LOG(INFO) << "Tensor image shape: " << image->shape(); + auto image = row["image"]; + MS_LOG(INFO) << "Tensor image shape: " << image.Shape(); iter->GetNextRow(&row); } @@ -1344,8 +1344,8 @@ TEST_F(MindDataTestPipeline, TestShuffleDataset) { uint64_t i = 0; while (row.size() != 0) { i++; - // auto image = row["image"]; - // MS_LOG(INFO) << "Tensor image shape: " << image->shape(); + auto image = row["image"]; + MS_LOG(INFO) << "Tensor image shape: " << image.Shape(); iter->GetNextRow(&row); } @@ -1380,8 +1380,8 @@ TEST_F(MindDataTestPipeline, TestSkipDataset) { uint64_t i = 0; while (row.size() != 0) { i++; - // auto image = row["image"]; - // MS_LOG(INFO) << "Tensor image shape: " << image->shape(); + auto image = row["image"]; + MS_LOG(INFO) << "Tensor image shape: " << image.Shape(); iter->GetNextRow(&row); } MS_LOG(INFO) << "Number of rows: " << i; @@ -1425,8 +1425,8 @@ TEST_F(MindDataTestPipeline, TestSkipTakeRepeat) { uint64_t i = 0; while (row.size() != 0) { i++; - // auto image = row["image"]; - // MS_LOG(INFO) << "Tensor image shape: " << image->shape(); + auto image = row["image"]; + MS_LOG(INFO) << "Tensor image shape: " << image.Shape(); iter->GetNextRow(&row); } MS_LOG(INFO) << "Number of rows: " << i; @@ -1497,8 +1497,8 @@ TEST_F(MindDataTestPipeline, TestTakeDatasetDefault) { uint64_t i = 0; while (row.size() != 0) { i++; - // auto image = row["image"]; - // MS_LOG(INFO) << "Tensor image shape: " << image->shape(); + auto image = row["image"]; + MS_LOG(INFO) << "Tensor image shape: " << image.Shape(); iter->GetNextRow(&row); } MS_LOG(INFO) << "Number of rows: " << i; @@ -1578,8 +1578,8 @@ TEST_F(MindDataTestPipeline, TestTakeDatasetNormal) { uint64_t i = 0; while (row.size() != 0) { i++; - // auto image = row["image"]; - // MS_LOG(INFO) << "Tensor image shape: " << image->shape(); + auto image = row["image"]; + MS_LOG(INFO) << "Tensor image shape: " << image.Shape(); iter->GetNextRow(&row); } MS_LOG(INFO) << "Number of rows: " << i; @@ -1632,8 +1632,8 @@ TEST_F(MindDataTestPipeline, TestTensorOpsAndMap) { uint64_t i = 0; while (row.size() != 0) { i++; - // auto image = row["image"]; - // MS_LOG(INFO) << "Tensor image shape: " << image->shape(); + auto image = row["image"]; + MS_LOG(INFO) << "Tensor image shape: " << image.Shape(); iter->GetNextRow(&row); } @@ -1751,8 +1751,8 @@ TEST_F(MindDataTestPipeline, TestZipSuccess) { uint64_t i = 0; while (row.size() != 0) { i++; - // auto image = row["image"]; - // MS_LOG(INFO) << "Tensor image shape: " << image->shape(); + auto image = row["image"]; + MS_LOG(INFO) << "Tensor image shape: " << image.Shape(); iter->GetNextRow(&row); } @@ -1843,8 +1843,8 @@ TEST_F(MindDataTestPipeline, TestZipSuccess2) { uint64_t i = 0; while (row.size() != 0) { i++; - // auto image = row["image"]; - // MS_LOG(INFO) << "Tensor image shape: " << image->shape(); + auto image = row["image"]; + MS_LOG(INFO) << "Tensor image shape: " << image.Shape(); iter->GetNextRow(&row); } diff --git a/tests/ut/cpp/dataset/c_api_dataset_randomdata_test.cc b/tests/ut/cpp/dataset/c_api_dataset_randomdata_test.cc index c5ce59fb30..ee4eccbc65 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_randomdata_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_randomdata_test.cc @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -54,10 +54,10 @@ TEST_F(MindDataTestPipeline, TestRandomDatasetBasic1) { // Check if RandomDataOp read correct columns uint64_t i = 0; while (row.size() != 0) { - // auto image = row["image"]; - // auto label = row["label"]; - // MS_LOG(INFO) << "Tensor image shape: " << image->shape(); - // MS_LOG(INFO) << "Tensor label shape: " << label->shape(); + auto image = row["image"]; + auto label = row["label"]; + MS_LOG(INFO) << "Tensor image shape: " << image.Shape(); + MS_LOG(INFO) << "Tensor label shape: " << label.Shape(); iter->GetNextRow(&row); i++; @@ -112,10 +112,10 @@ TEST_F(MindDataTestPipeline, TestRandomDatasetBasicWithPipeline) { // Check if RandomDataOp read correct columns uint64_t i = 0; while (row.size() != 0) { - // auto image = row["image"]; - // auto label = row["label"]; - // MS_LOG(INFO) << "Tensor image shape: " << image->shape(); - // MS_LOG(INFO) << "Tensor label shape: " << label->shape(); + auto image = row["image"]; + auto label = row["label"]; + MS_LOG(INFO) << "Tensor image shape: " << image.Shape(); + MS_LOG(INFO) << "Tensor label shape: " << label.Shape(); iter->GetNextRow(&row); i++; @@ -205,47 +205,52 @@ TEST_F(MindDataTestPipeline, TestRandomDatasetBasic3) { std::unordered_map row; iter->GetNextRow(&row); + std::vector expect_num = {1}; + std::vector expect_1d = {2}; + std::vector expect_2d = {2, 2}; + std::vector expect_3d = {2, 2, 2}; + // Check if RandomDataOp read correct columns uint64_t i = 0; while (row.size() != 0) { - // auto col_sint16 = row["col_sint16"]; - // auto col_sint32 = row["col_sint32"]; - // auto col_sint64 = row["col_sint64"]; - // auto col_float = row["col_float"]; - // auto col_1d = row["col_1d"]; - // auto col_2d = row["col_2d"]; - // auto col_3d = row["col_3d"]; - // auto col_binary = row["col_binary"]; - - // // validate shape - // ASSERT_EQ(col_sint16->shape(), TensorShape({1})); - // ASSERT_EQ(col_sint32->shape(), TensorShape({1})); - // ASSERT_EQ(col_sint64->shape(), TensorShape({1})); - // ASSERT_EQ(col_float->shape(), TensorShape({1})); - // ASSERT_EQ(col_1d->shape(), TensorShape({2})); - // ASSERT_EQ(col_2d->shape(), TensorShape({2, 2})); - // ASSERT_EQ(col_3d->shape(), TensorShape({2, 2, 2})); - // ASSERT_EQ(col_binary->shape(), TensorShape({1})); - - // // validate Rank - // ASSERT_EQ(col_sint16->Rank(), 1); - // ASSERT_EQ(col_sint32->Rank(), 1); - // ASSERT_EQ(col_sint64->Rank(), 1); - // ASSERT_EQ(col_float->Rank(), 1); - // ASSERT_EQ(col_1d->Rank(), 1); - // ASSERT_EQ(col_2d->Rank(), 2); - // ASSERT_EQ(col_3d->Rank(), 3); - // ASSERT_EQ(col_binary->Rank(), 1); - - // // validate type - // ASSERT_EQ(col_sint16->type(), DataType::DE_INT16); - // ASSERT_EQ(col_sint32->type(), DataType::DE_INT32); - // ASSERT_EQ(col_sint64->type(), DataType::DE_INT64); - // ASSERT_EQ(col_float->type(), DataType::DE_FLOAT32); - // ASSERT_EQ(col_1d->type(), DataType::DE_INT64); - // ASSERT_EQ(col_2d->type(), DataType::DE_INT64); - // ASSERT_EQ(col_3d->type(), DataType::DE_INT64); - // ASSERT_EQ(col_binary->type(), DataType::DE_UINT8); + auto col_sint16 = row["col_sint16"]; + auto col_sint32 = row["col_sint32"]; + auto col_sint64 = row["col_sint64"]; + auto col_float = row["col_float"]; + auto col_1d = row["col_1d"]; + auto col_2d = row["col_2d"]; + auto col_3d = row["col_3d"]; + auto col_binary = row["col_binary"]; + + // Validate shape + ASSERT_EQ(col_sint16.Shape(), expect_num); + ASSERT_EQ(col_sint32.Shape(), expect_num); + ASSERT_EQ(col_sint64.Shape(), expect_num); + ASSERT_EQ(col_float.Shape(), expect_num); + ASSERT_EQ(col_1d.Shape(), expect_1d); + ASSERT_EQ(col_2d.Shape(), expect_2d); + ASSERT_EQ(col_3d.Shape(), expect_3d); + ASSERT_EQ(col_binary.Shape(), expect_num); + + // Validate Rank + ASSERT_EQ(col_sint16.Shape().size(), 1); + ASSERT_EQ(col_sint32.Shape().size(), 1); + ASSERT_EQ(col_sint64.Shape().size(), 1); + ASSERT_EQ(col_float.Shape().size(), 1); + ASSERT_EQ(col_1d.Shape().size(), 1); + ASSERT_EQ(col_2d.Shape().size(), 2); + ASSERT_EQ(col_3d.Shape().size(), 3); + ASSERT_EQ(col_binary.Shape().size(), 1); + + // Validate type + ASSERT_EQ(col_sint16.DataType(), mindspore::DataType::kNumberTypeInt16); + ASSERT_EQ(col_sint32.DataType(), mindspore::DataType::kNumberTypeInt32); + ASSERT_EQ(col_sint64.DataType(), mindspore::DataType::kNumberTypeInt64); + ASSERT_EQ(col_float.DataType(), mindspore::DataType::kNumberTypeFloat32); + ASSERT_EQ(col_1d.DataType(), mindspore::DataType::kNumberTypeInt64); + ASSERT_EQ(col_2d.DataType(), mindspore::DataType::kNumberTypeInt64); + ASSERT_EQ(col_3d.DataType(), mindspore::DataType::kNumberTypeInt64); + ASSERT_EQ(col_binary.DataType(), mindspore::DataType::kNumberTypeUInt8); iter->GetNextRow(&row); i++; @@ -282,47 +287,52 @@ TEST_F(MindDataTestPipeline, TestRandomDatasetBasic4) { std::unordered_map row; iter->GetNextRow(&row); + std::vector expect_num = {1}; + std::vector expect_1d = {2}; + std::vector expect_2d = {2, 2}; + std::vector expect_3d = {2, 2, 2}; + // Check if RandomDataOp read correct columns uint64_t i = 0; while (row.size() != 0) { - // auto col_sint16 = row["col_sint16"]; - // auto col_sint32 = row["col_sint32"]; - // auto col_sint64 = row["col_sint64"]; - // auto col_float = row["col_float"]; - // auto col_1d = row["col_1d"]; - // auto col_2d = row["col_2d"]; - // auto col_3d = row["col_3d"]; - // auto col_binary = row["col_binary"]; - - // // validate shape - // ASSERT_EQ(col_sint16->shape(), TensorShape({1})); - // ASSERT_EQ(col_sint32->shape(), TensorShape({1})); - // ASSERT_EQ(col_sint64->shape(), TensorShape({1})); - // ASSERT_EQ(col_float->shape(), TensorShape({1})); - // ASSERT_EQ(col_1d->shape(), TensorShape({2})); - // ASSERT_EQ(col_2d->shape(), TensorShape({2, 2})); - // ASSERT_EQ(col_3d->shape(), TensorShape({2, 2, 2})); - // ASSERT_EQ(col_binary->shape(), TensorShape({1})); - - // // validate Rank - // ASSERT_EQ(col_sint16->Rank(), 1); - // ASSERT_EQ(col_sint32->Rank(), 1); - // ASSERT_EQ(col_sint64->Rank(), 1); - // ASSERT_EQ(col_float->Rank(), 1); - // ASSERT_EQ(col_1d->Rank(), 1); - // ASSERT_EQ(col_2d->Rank(), 2); - // ASSERT_EQ(col_3d->Rank(), 3); - // ASSERT_EQ(col_binary->Rank(), 1); - - // // validate type - // ASSERT_EQ(col_sint16->type(), DataType::DE_INT16); - // ASSERT_EQ(col_sint32->type(), DataType::DE_INT32); - // ASSERT_EQ(col_sint64->type(), DataType::DE_INT64); - // ASSERT_EQ(col_float->type(), DataType::DE_FLOAT32); - // ASSERT_EQ(col_1d->type(), DataType::DE_INT64); - // ASSERT_EQ(col_2d->type(), DataType::DE_INT64); - // ASSERT_EQ(col_3d->type(), DataType::DE_INT64); - // ASSERT_EQ(col_binary->type(), DataType::DE_UINT8); + auto col_sint16 = row["col_sint16"]; + auto col_sint32 = row["col_sint32"]; + auto col_sint64 = row["col_sint64"]; + auto col_float = row["col_float"]; + auto col_1d = row["col_1d"]; + auto col_2d = row["col_2d"]; + auto col_3d = row["col_3d"]; + auto col_binary = row["col_binary"]; + + // Validate shape + ASSERT_EQ(col_sint16.Shape(), expect_num); + ASSERT_EQ(col_sint32.Shape(), expect_num); + ASSERT_EQ(col_sint64.Shape(), expect_num); + ASSERT_EQ(col_float.Shape(), expect_num); + ASSERT_EQ(col_1d.Shape(), expect_1d); + ASSERT_EQ(col_2d.Shape(), expect_2d); + ASSERT_EQ(col_3d.Shape(), expect_3d); + ASSERT_EQ(col_binary.Shape(), expect_num); + + // Validate Rank + ASSERT_EQ(col_sint16.Shape().size(), 1); + ASSERT_EQ(col_sint32.Shape().size(), 1); + ASSERT_EQ(col_sint64.Shape().size(), 1); + ASSERT_EQ(col_float.Shape().size(), 1); + ASSERT_EQ(col_1d.Shape().size(), 1); + ASSERT_EQ(col_2d.Shape().size(), 2); + ASSERT_EQ(col_3d.Shape().size(), 3); + ASSERT_EQ(col_binary.Shape().size(), 1); + + // Validate type + ASSERT_EQ(col_sint16.DataType(), mindspore::DataType::kNumberTypeInt16); + ASSERT_EQ(col_sint32.DataType(), mindspore::DataType::kNumberTypeInt32); + ASSERT_EQ(col_sint64.DataType(), mindspore::DataType::kNumberTypeInt64); + ASSERT_EQ(col_float.DataType(), mindspore::DataType::kNumberTypeFloat32); + ASSERT_EQ(col_1d.DataType(), mindspore::DataType::kNumberTypeInt64); + ASSERT_EQ(col_2d.DataType(), mindspore::DataType::kNumberTypeInt64); + ASSERT_EQ(col_3d.DataType(), mindspore::DataType::kNumberTypeInt64); + ASSERT_EQ(col_binary.DataType(), mindspore::DataType::kNumberTypeUInt8); iter->GetNextRow(&row); i++; @@ -359,29 +369,32 @@ TEST_F(MindDataTestPipeline, TestRandomDatasetBasic5) { std::unordered_map row; iter->GetNextRow(&row); + std::vector expect_num = {1}; + std::vector expect_1d = {2}; + // Check if RandomDataOp read correct columns uint64_t i = 0; while (row.size() != 0) { EXPECT_EQ(row.size(), 3); - // auto col_sint32 = row["col_sint32"]; - // auto col_sint64 = row["col_sint64"]; - // auto col_1d = row["col_1d"]; + auto col_sint32 = row["col_sint32"]; + auto col_sint64 = row["col_sint64"]; + auto col_1d = row["col_1d"]; - // // validate shape - // ASSERT_EQ(col_sint32->shape(), TensorShape({1})); - // ASSERT_EQ(col_sint64->shape(), TensorShape({1})); - // ASSERT_EQ(col_1d->shape(), TensorShape({2})); + // Validate shape + ASSERT_EQ(col_sint32.Shape(), expect_num); + ASSERT_EQ(col_sint64.Shape(), expect_num); + ASSERT_EQ(col_1d.Shape(), expect_1d); - // // validate Rank - // ASSERT_EQ(col_sint32->Rank(), 1); - // ASSERT_EQ(col_sint64->Rank(), 1); - // ASSERT_EQ(col_1d->Rank(), 1); + // Validate Rank + ASSERT_EQ(col_sint32.Shape().size(), 1); + ASSERT_EQ(col_sint64.Shape().size(), 1); + ASSERT_EQ(col_1d.Shape().size(), 1); - // // validate type - // ASSERT_EQ(col_sint32->type(), DataType::DE_INT32); - // ASSERT_EQ(col_sint64->type(), DataType::DE_INT64); - // ASSERT_EQ(col_1d->type(), DataType::DE_INT64); + // Validate type + ASSERT_EQ(col_sint32.DataType(), mindspore::DataType::kNumberTypeInt32); + ASSERT_EQ(col_sint64.DataType(), mindspore::DataType::kNumberTypeInt64); + ASSERT_EQ(col_1d.DataType(), mindspore::DataType::kNumberTypeInt64); iter->GetNextRow(&row); i++; diff --git a/tests/ut/cpp/dataset/c_api_dataset_textfile_test.cc b/tests/ut/cpp/dataset/c_api_dataset_textfile_test.cc index 5dc1857410..23ec963bd1 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_textfile_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_textfile_test.cc @@ -1,5 +1,5 @@ /** - * Copyright 2020 Huawei Technologies Co., Ltd + * Copyright 2020-2021 Huawei Technologies Co., Ltd * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -58,8 +58,8 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetBasic) { uint64_t i = 0; while (row.size() != 0) { - // auto text = row["text"]; - // MS_LOG(INFO) << "Tensor text shape: " << text->shape(); + auto text = row["text"]; + MS_LOG(INFO) << "Tensor text shape: " << text.Shape(); // std::string_view sv; // text->GetItemAt(&sv, {0}); // std::string ss(sv); @@ -128,8 +128,8 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetBasicWithPipeline) { uint64_t i = 0; while (row.size() != 0) { - // auto text = row["text"]; - // MS_LOG(INFO) << "Tensor text shape: " << text->shape(); + auto text = row["text"]; + MS_LOG(INFO) << "Tensor text shape: " << text.Shape(); i++; iter->GetNextRow(&row); } @@ -315,8 +315,8 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleFalse1A) { uint64_t i = 0; while (row.size() != 0) { - // auto text = row["text"]; - // MS_LOG(INFO) << "Tensor text shape: " << text->shape(); + auto text = row["text"]; + MS_LOG(INFO) << "Tensor text shape: " << text.Shape(); // std::string_view sv; // text->GetItemAt(&sv, {0}); // std::string ss(sv); @@ -373,8 +373,8 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleFalse1B) { uint64_t i = 0; while (row.size() != 0) { - // auto text = row["text"]; - // MS_LOG(INFO) << "Tensor text shape: " << text->shape(); + auto text = row["text"]; + MS_LOG(INFO) << "Tensor text shape: " << text.Shape(); // std::string_view sv; // text->GetItemAt(&sv, {0}); // std::string ss(sv); @@ -430,8 +430,8 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleFalse4Shard) { uint64_t i = 0; while (row.size() != 0) { - // auto text = row["text"]; - // MS_LOG(INFO) << "Tensor text shape: " << text->shape(); + auto text = row["text"]; + MS_LOG(INFO) << "Tensor text shape: " << text.Shape(); // std::string_view sv; // text->GetItemAt(&sv, {0}); // std::string ss(sv); @@ -490,8 +490,8 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleFiles1A) { uint64_t i = 0; while (row.size() != 0) { - // auto text = row["text"]; - // MS_LOG(INFO) << "Tensor text shape: " << text->shape(); + auto text = row["text"]; + MS_LOG(INFO) << "Tensor text shape: " << text.Shape(); // std::string_view sv; // text->GetItemAt(&sv, {0}); // std::string ss(sv); @@ -550,8 +550,8 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleFiles1B) { uint64_t i = 0; while (row.size() != 0) { - // auto text = row["text"]; - // MS_LOG(INFO) << "Tensor text shape: " << text->shape(); + auto text = row["text"]; + MS_LOG(INFO) << "Tensor text shape: " << text.Shape(); // std::string_view sv; // text->GetItemAt(&sv, {0}); // std::string ss(sv); @@ -609,8 +609,8 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleFiles4) { uint64_t i = 0; while (row.size() != 0) { - // auto text = row["text"]; - // MS_LOG(INFO) << "Tensor text shape: " << text->shape(); + auto text = row["text"]; + MS_LOG(INFO) << "Tensor text shape: " << text.Shape(); // std::string_view sv; // text->GetItemAt(&sv, {0}); // std::string ss(sv); @@ -664,8 +664,8 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleGlobal1A) { uint64_t i = 0; while (row.size() != 0) { - // auto text = row["text"]; - // MS_LOG(INFO) << "Tensor text shape: " << text->shape(); + auto text = row["text"]; + MS_LOG(INFO) << "Tensor text shape: " << text.Shape(); // std::string_view sv; // text->GetItemAt(&sv, {0}); // std::string ss(sv); @@ -722,8 +722,8 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleGlobal1B) { uint64_t i = 0; while (row.size() != 0) { - // auto text = row["text"]; - // MS_LOG(INFO) << "Tensor text shape: " << text->shape(); + auto text = row["text"]; + MS_LOG(INFO) << "Tensor text shape: " << text.Shape(); // std::string_view sv; // text->GetItemAt(&sv, {0}); // std::string ss(sv); @@ -780,8 +780,8 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleGlobal4) { uint64_t i = 0; while (row.size() != 0) { - // auto text = row["text"]; - // MS_LOG(INFO) << "Tensor text shape: " << text->shape(); + auto text = row["text"]; + MS_LOG(INFO) << "Tensor text shape: " << text.Shape(); // std::string_view sv; // text->GetItemAt(&sv, {0}); // std::string ss(sv); diff --git a/tests/ut/cpp/dataset/c_api_dataset_tfrecord_test.cc b/tests/ut/cpp/dataset/c_api_dataset_tfrecord_test.cc index 4512bf448c..3f702e5e8e 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_tfrecord_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_tfrecord_test.cc @@ -70,9 +70,9 @@ TEST_F(MindDataTestPipeline, TestTFRecordDatasetBasic) { uint64_t i = 0; while (row.size() != 0) { - // auto image = row["image"]; + auto image = row["image"]; - // MS_LOG(INFO) << "Tensor image shape: " << image->shape(); + MS_LOG(INFO) << "Tensor image shape: " << image.Shape(); iter->GetNextRow(&row); i++; } @@ -279,23 +279,30 @@ TEST_F(MindDataTestPipeline, TestTFRecordDatasetSchemaObj) { EXPECT_NE(row.find("col_float"), row.end()); EXPECT_NE(row.find("col_2d"), row.end()); + std::vector expect_num = {1}; + std::vector expect_2d = {2, 2}; + uint64_t i = 0; while (row.size() != 0) { - // auto col_sint16 = row["col_sint16"]; - // auto col_float = row["col_float"]; - // auto col_2d = row["col_2d"]; + auto col_sint16 = row["col_sint16"]; + auto col_float = row["col_float"]; + auto col_2d = row["col_2d"]; + + // Validate shape + ASSERT_EQ(col_sint16.Shape(), expect_num); + ASSERT_EQ(col_float.Shape(), expect_num); + ASSERT_EQ(col_2d.Shape(), expect_2d); - // EXPECT_EQ(col_sint16->shape(), TensorShape({1})); - // EXPECT_EQ(col_float->shape(), TensorShape({1})); - // EXPECT_EQ(col_2d->shape(), TensorShape({2, 2})); + // Validate Rank + ASSERT_EQ(col_sint16.Shape().size(), 1); + ASSERT_EQ(col_float.Shape().size(), 1); + ASSERT_EQ(col_2d.Shape().size(), 2); - // EXPECT_EQ(col_sint16->Rank(), 1); - // EXPECT_EQ(col_float->Rank(), 1); - // EXPECT_EQ(col_2d->Rank(), 2); + // Validate type + ASSERT_EQ(col_sint16.DataType(), mindspore::DataType::kNumberTypeInt16); + ASSERT_EQ(col_float.DataType(), mindspore::DataType::kNumberTypeFloat32); + ASSERT_EQ(col_2d.DataType(), mindspore::DataType::kNumberTypeInt64); - // EXPECT_EQ(col_sint16->type(), DataType::DE_INT16); - // EXPECT_EQ(col_float->type(), DataType::DE_FLOAT32); - // EXPECT_EQ(col_2d->type(), DataType::DE_INT64); iter->GetNextRow(&row); i++; } @@ -331,11 +338,11 @@ TEST_F(MindDataTestPipeline, TestTFRecordDatasetNoSchema) { uint64_t i = 0; while (row.size() != 0) { - // auto image = row["image"]; - // auto label = row["label"]; + auto image = row["image"]; + auto label = row["label"]; - // MS_LOG(INFO) << "Shape of column [image]:" << image->shape(); - // MS_LOG(INFO) << "Shape of column [label]:" << label->shape(); + MS_LOG(INFO) << "Shape of column [image]:" << image.Shape(); + MS_LOG(INFO) << "Shape of column [label]:" << label.Shape(); iter->GetNextRow(&row); i++; } @@ -486,16 +493,16 @@ TEST_F(MindDataTestPipeline, TestIncorrectTFSchemaObject) { EXPECT_NE(ds, nullptr); auto itr = ds->CreateIterator(); EXPECT_NE(itr, nullptr); - // TensorMap mp; - // this will fail due to the incorrect schema used - // EXPECT_FALSE(itr->GetNextRow(&mp)); + MSTensorMap mp; + // This will fail due to the incorrect schema used + EXPECT_ERROR(itr->GetNextRow(&mp)); } TEST_F(MindDataTestPipeline, TestIncorrectTFrecordFile) { std::string path = datasets_root_path_ + "/test_tf_file_3_images2/datasetSchema.json"; std::shared_ptr ds = TFRecord({path}); EXPECT_NE(ds, nullptr); - // the tf record file is incorrect, hence validate param will fail + // The tf record file is incorrect, hence validate param will fail auto itr = ds->CreateIterator(); EXPECT_EQ(itr, nullptr); } diff --git a/tests/ut/cpp/dataset/c_api_dataset_voc_test.cc b/tests/ut/cpp/dataset/c_api_dataset_voc_test.cc index cd24bd650f..cfd227bfc2 100644 --- a/tests/ut/cpp/dataset/c_api_dataset_voc_test.cc +++ b/tests/ut/cpp/dataset/c_api_dataset_voc_test.cc @@ -55,10 +55,10 @@ TEST_F(MindDataTestPipeline, TestVOCClassIndex) { // uint32_t expect[] = {9, 9, 9, 1, 1, 0}; uint64_t i = 0; while (row.size() != 0) { - // auto image = row["image"]; - // auto label = row["label"]; - // MS_LOG(INFO) << "Tensor image shape: " << image->shape(); - // MS_LOG(INFO) << "Tensor label shape: " << label->shape(); + auto image = row["image"]; + auto label = row["label"]; + MS_LOG(INFO) << "Tensor image shape: " << image.Shape(); + MS_LOG(INFO) << "Tensor label shape: " << label.Shape(); // expect_label->SetItemAt({0, 0}, expect[i]); // EXPECT_EQ(*label, *expect_label); @@ -137,10 +137,10 @@ TEST_F(MindDataTestPipeline, TestVOCDetection) { // uint32_t expect_num[] = {5, 5, 4, 3}; uint64_t i = 0; while (row.size() != 0) { - // auto image = row["image"]; - // auto label = row["label"]; - // MS_LOG(INFO) << "Tensor image shape: " << image->shape(); - // MS_LOG(INFO) << "Tensor label shape: " << label->shape(); + auto image = row["image"]; + auto label = row["label"]; + MS_LOG(INFO) << "Tensor image shape: " << image.Shape(); + MS_LOG(INFO) << "Tensor label shape: " << label.Shape(); // std::shared_ptr expect_image; // Tensor::CreateFromFile(folder_path + "/JPEGImages/" + expect_file[i] + ".jpg", &expect_image); @@ -210,10 +210,10 @@ TEST_F(MindDataTestPipeline, TestVOCSegmentation) { // std::string expect_file[] = {"32", "33", "39", "32", "33", "39"}; uint64_t i = 0; while (row.size() != 0) { - // auto image = row["image"]; - // auto target = row["target"]; - // MS_LOG(INFO) << "Tensor image shape: " << image->shape(); - // MS_LOG(INFO) << "Tensor target shape: " << target->shape(); + auto image = row["image"]; + auto target = row["target"]; + MS_LOG(INFO) << "Tensor image shape: " << image.Shape(); + MS_LOG(INFO) << "Tensor target shape: " << target.Shape(); // std::shared_ptr expect_image; // Tensor::CreateFromFile(folder_path + "/JPEGImages/" + expect_file[i] + ".jpg", &expect_image); diff --git a/tests/ut/cpp/dataset/c_api_datasets_test.cc b/tests/ut/cpp/dataset/c_api_datasets_test.cc index 1bb1374bb2..bc9f772d40 100644 --- a/tests/ut/cpp/dataset/c_api_datasets_test.cc +++ b/tests/ut/cpp/dataset/c_api_datasets_test.cc @@ -93,10 +93,10 @@ TEST_F(MindDataTestPipeline, TestCelebADefault) { // Check if CelebAOp read correct images/attr uint64_t i = 0; while (row.size() != 0) { - // auto image = row["image"]; - // auto attr = row["attr"]; - // MS_LOG(INFO) << "Tensor image shape: " << image->shape(); - // MS_LOG(INFO) << "Tensor attr shape: " << attr->shape(); + auto image = row["image"]; + auto attr = row["attr"]; + MS_LOG(INFO) << "Tensor image shape: " << image.Shape(); + MS_LOG(INFO) << "Tensor attr shape: " << attr.Shape(); iter->GetNextRow(&row); i++; diff --git a/tests/ut/cpp/dataset/ir_vision_test.cc b/tests/ut/cpp/dataset/ir_vision_test.cc index 13ef9bd3f7..ccdac3c207 100644 --- a/tests/ut/cpp/dataset/ir_vision_test.cc +++ b/tests/ut/cpp/dataset/ir_vision_test.cc @@ -329,7 +329,7 @@ TEST_F(MindDataTestIRVision, TestSoftDvppDecodeResizeJpegFail) { Status rc; - // CSoftDvppDecodeResizeJpeg: size must be a vector of one or two values + // SoftDvppDecodeResizeJpeg: size must be a vector of one or two values std::shared_ptr soft_dvpp_decode_resize_jpeg_op1(new vision::SoftDvppDecodeResizeJpegOperation({})); rc = soft_dvpp_decode_resize_jpeg_op1->ValidateParams(); EXPECT_ERROR(rc);