m53297601
/
mindspore
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

!13078 dataset UT: Reinstate INFO logging and data verification - part 2

Browse Source 
From: @cathwong
Reviewed-by: 
Signed-off-by:
pull/13078/MERGE
mindspore-ci-bot 4 years ago committed by Gitee
parent 654be6216e 95bbaaf9f6
commit 483bb9de60
10 changed files with 294 additions and 234 deletions
Show Stats Download Patch File Download Diff File

58
tests/ut/cpp/dataset/c_api_cache_test.cc
Unescape View File

@ -105,8 +105,8 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheImageFolderCApi) {
uint64_t i = 0; uint64_t i = 0;
while (row.size() != 0) { while (row.size() != 0) {
i++; i++;
// auto image = row["image"]; auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape(); MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row); iter->GetNextRow(&row);
} }
@ -148,8 +148,8 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheCocoCApi) {
uint64_t i = 0; uint64_t i = 0;
while (row.size() != 0) { while (row.size() != 0) {
i++; i++;
// auto image = row["image"]; auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape(); MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row); iter->GetNextRow(&row);
} }
@ -189,8 +189,8 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheMnistCApi) {
uint64_t i = 0; uint64_t i = 0;
while (row.size() != 0) { while (row.size() != 0) {
i++; i++;
// auto image = row["image"]; auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape(); MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row); iter->GetNextRow(&row);
} }
@ -231,8 +231,8 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheCelebaCApi) {
uint64_t i = 0; uint64_t i = 0;
while (row.size() != 0) { while (row.size() != 0) {
i++; i++;
// auto image = row["image"]; auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape(); MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row); iter->GetNextRow(&row);
} }
@ -272,8 +272,8 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheManifestCApi) {
uint64_t i = 0; uint64_t i = 0;
while (row.size() != 0) { while (row.size() != 0) {
i++; i++;
// auto image = row["image"]; auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape(); MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row); iter->GetNextRow(&row);
} }
@ -313,8 +313,8 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheCifar10CApi) {
uint64_t i = 0; uint64_t i = 0;
while (row.size() != 0) { while (row.size() != 0) {
i++; i++;
// auto image = row["image"]; auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape(); MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row); iter->GetNextRow(&row);
} }
@ -354,8 +354,8 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheCifar100CApi) {
uint64_t i = 0; uint64_t i = 0;
while (row.size() != 0) { while (row.size() != 0) {
i++; i++;
// auto image = row["image"]; auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape(); MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row); iter->GetNextRow(&row);
} }
@ -396,8 +396,8 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheVocCApi) {
uint64_t i = 0; uint64_t i = 0;
while (row.size() != 0) { while (row.size() != 0) {
i++; i++;
// auto image = row["image"]; auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape(); MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row); iter->GetNextRow(&row);
} }
@ -523,8 +523,8 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheTFRecordCApi1) {
uint64_t i = 0; uint64_t i = 0;
while (row.size() != 0) { while (row.size() != 0) {
i++; i++;
// auto image = row["image"]; auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape(); MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row); iter->GetNextRow(&row);
} }
@ -574,8 +574,8 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheTFRecordCApi2) {
uint64_t i = 0; uint64_t i = 0;
while (row.size() != 0) { while (row.size() != 0) {
i++; i++;
// auto image = row["image"]; auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape(); MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row); iter->GetNextRow(&row);
} }
@ -621,8 +621,8 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCacheTFRecordCApi3) {
uint64_t i = 0; uint64_t i = 0;
while (row.size() != 0) { while (row.size() != 0) {
i++; i++;
// auto image = row["image"]; auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape(); MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row); iter->GetNextRow(&row);
} }
@ -789,8 +789,8 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCApiCacheShare1) {
uint64_t i = 0; uint64_t i = 0;
while (row.size() != 0) { while (row.size() != 0) {
i++; i++;
// auto image = row["image"]; auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape(); MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter1->GetNextRow(&row); iter1->GetNextRow(&row);
} }
EXPECT_EQ(i, 2); EXPECT_EQ(i, 2);
@ -806,8 +806,8 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCApiCacheShare1) {
i = 0; i = 0;
while (row.size() != 0) { while (row.size() != 0) {
i++; i++;
// auto image = row["image"]; auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape(); MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter2->GetNextRow(&row); iter2->GetNextRow(&row);
} }
EXPECT_EQ(i, 2); EXPECT_EQ(i, 2);
@ -844,7 +844,7 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCApiCacheShare2) {
uint64_t i = 0; uint64_t i = 0;
while (row.size() != 0) { while (row.size() != 0) {
i++; i++;
// auto image = row["image"]; auto image = row["image"];
iter1->GetNextRow(&row); iter1->GetNextRow(&row);
} }
EXPECT_EQ(i, 2); EXPECT_EQ(i, 2);
@ -860,7 +860,7 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCApiCacheShare2) {
i = 0; i = 0;
while (row.size() != 0) { while (row.size() != 0) {
i++; i++;
// auto image = row["image"]; auto image = row["image"];
iter2->GetNextRow(&row); iter2->GetNextRow(&row);
} }
EXPECT_EQ(i, 2); EXPECT_EQ(i, 2);
@ -894,7 +894,7 @@ TEST_F(MindDataTestCacheOp, DISABLED_TestCApiCacheShareFailure1) {
uint64_t i = 0; uint64_t i = 0;
while (row.size() != 0) { while (row.size() != 0) {
i++; i++;
// auto image = row["image"]; auto image = row["image"];
iter1->GetNextRow(&row); iter1->GetNextRow(&row);
} }
EXPECT_EQ(i, 2); EXPECT_EQ(i, 2);

238
tests/ut/cpp/dataset/c_api_dataset_coco_test.cc
View File

File diff suppressed because it is too large Load Diff

45
tests/ut/cpp/dataset/c_api_dataset_minddata_test.cc
Unescape View File

@ -48,6 +48,7 @@ TEST_F(MindDataTestPipeline, TestMindDataSuccess1) {
i++; i++;
// auto image = row["file_name"]; // auto image = row["file_name"];
// MS_LOG(INFO) << "Tensor image file name: " << *image; // MS_LOG(INFO) << "Tensor image file name: " << *image;
iter->GetNextRow(&row); iter->GetNextRow(&row);
} }
@ -196,14 +197,16 @@ TEST_F(MindDataTestPipeline, TestMindDataSuccess5) {
std::unordered_map<std::string, mindspore::MSTensor> row; std::unordered_map<std::string, mindspore::MSTensor> row;
iter->GetNextRow(&row); iter->GetNextRow(&row);
std::shared_ptr<Tensor> de_expect_item;
ASSERT_OK(Tensor::CreateScalar((int64_t)0, &de_expect_item));
mindspore::MSTensor expect_item = mindspore::MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_expect_item));
uint64_t i = 0; uint64_t i = 0;
while (row.size() != 0) { while (row.size() != 0) {
i++; i++;
// auto label = row["label"]; auto label = row["label"];
// mindspore::MSTensor expected_item; EXPECT_MSTENSOR_EQ(label, expect_item);
// Tensor::CreateScalar((int64_t)0, &expected_item);
// EXPECT_EQ(*expected_item, *label);
iter->GetNextRow(&row); iter->GetNextRow(&row);
} }
@ -302,17 +305,19 @@ TEST_F(MindDataTestPipeline, TestMindDataSuccess7) {
std::unordered_map<std::string, mindspore::MSTensor> row; std::unordered_map<std::string, mindspore::MSTensor> row;
iter->GetNextRow(&row); iter->GetNextRow(&row);
std::shared_ptr<Tensor> de_expect_item;
ASSERT_OK(Tensor::CreateScalar((int64_t)999, &de_expect_item));
mindspore::MSTensor expect_item = mindspore::MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_expect_item));
uint64_t i = 0; uint64_t i = 0;
while (row.size() != 0) { while (row.size() != 0) {
i++; i++;
// auto image = row["file_name"]; // auto image = row["file_name"];
// auto label = row["label"]; auto label = row["label"];
// MS_LOG(INFO) << "Tensor file name: " << *image; // MS_LOG(INFO) << "Tensor file name: " << *image;
// MS_LOG(INFO) << "Tensor label: " << *label; // MS_LOG(INFO) << "Tensor label: " << *label;
// mindspore::MSTensor expected_item; EXPECT_MSTENSOR_EQ(label, expect_item);
// Tensor::CreateScalar((int64_t)999, &expected_item);
// EXPECT_EQ(*expected_item, *label);
iter->GetNextRow(&row); iter->GetNextRow(&row);
} }
@ -339,8 +344,8 @@ TEST_F(MindDataTestPipeline, TestMindDataSuccess8) {
MindData(file_list, {"file_name", "label"}, std::make_shared<SequentialSampler>(), pad, 4); MindData(file_list, {"file_name", "label"}, std::make_shared<SequentialSampler>(), pad, 4);
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);
std::vector<DataType> types = ToDETypes(ds->GetOutputTypes()); std::vector<mindspore::dataset::DataType> types = ToDETypes(ds->GetOutputTypes());
std::vector<TensorShape> shapes = ToTensorShapeVec(ds->GetOutputShapes()); std::vector<mindspore::dataset::TensorShape> shapes = ToTensorShapeVec(ds->GetOutputShapes());
std::vector<std::string> column_names = {"file_name", "label"}; std::vector<std::string> column_names = {"file_name", "label"};
EXPECT_EQ(types.size(), 2); EXPECT_EQ(types.size(), 2);
EXPECT_EQ(types[0].ToString(), "string"); EXPECT_EQ(types[0].ToString(), "string");
@ -371,17 +376,19 @@ TEST_F(MindDataTestPipeline, TestMindDataSuccess8) {
std::unordered_map<std::string, mindspore::MSTensor> row; std::unordered_map<std::string, mindspore::MSTensor> row;
iter->GetNextRow(&row); iter->GetNextRow(&row);
std::shared_ptr<Tensor> de_expect_item;
ASSERT_OK(Tensor::CreateScalar((int64_t)999, &de_expect_item));
mindspore::MSTensor expect_item = mindspore::MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_expect_item));
uint64_t i = 0; uint64_t i = 0;
while (row.size() != 0) { while (row.size() != 0) {
i++; i++;
// auto image = row["file_name"]; // auto image = row["file_name"];
// auto label = row["label"]; auto label = row["label"];
// MS_LOG(INFO) << "Tensor file name: " << *image; // MS_LOG(INFO) << "Tensor file name: " << *image;
// MS_LOG(INFO) << "Tensor label: " << *label; // MS_LOG(INFO) << "Tensor label: " << *label;
// mindspore::MSTensor expected_item; EXPECT_MSTENSOR_EQ(label, expect_item);
// Tensor::CreateScalar((int64_t)999, &expected_item);
// EXPECT_EQ(*expected_item, *label);
iter->GetNextRow(&row); iter->GetNextRow(&row);
} }
@ -444,15 +451,17 @@ TEST_F(MindDataTestPipeline, TestMindDataSuccess9) {
std::unordered_map<std::string, mindspore::MSTensor> row; std::unordered_map<std::string, mindspore::MSTensor> row;
iter->GetNextRow(&row); iter->GetNextRow(&row);
std::shared_ptr<Tensor> de_expect_item;
ASSERT_OK(Tensor::CreateScalar((int64_t)999, &de_expect_item));
mindspore::MSTensor expect_item = mindspore::MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_expect_item));
uint64_t i = 0; uint64_t i = 0;
while (row.size() != 0) { while (row.size() != 0) {
i++; i++;
// auto label = row["label"]; auto label = row["label"];
// MS_LOG(INFO) << "Tensor label: " << *label; // MS_LOG(INFO) << "Tensor label: " << *label;
// mindspore::MSTensor expected_item; EXPECT_MSTENSOR_EQ(label, expect_item);
// Tensor::CreateScalar((int64_t)999, &expected_item);
// EXPECT_EQ(*expected_item, *label);
iter->GetNextRow(&row); iter->GetNextRow(&row);
} }

16
tests/ut/cpp/dataset/c_api_dataset_randomdata_test.cc
Unescape View File

@ -51,7 +51,7 @@ TEST_F(MindDataTestPipeline, TestRandomDatasetBasic1) {
std::unordered_map<std::string, mindspore::MSTensor> row; std::unordered_map<std::string, mindspore::MSTensor> row;
iter->GetNextRow(&row); iter->GetNextRow(&row);
// Check if RandomDataOp read correct columns // Check if RandomData() read correct columns
uint64_t i = 0; uint64_t i = 0;
while (row.size() != 0) { while (row.size() != 0) {
auto image = row["image"]; auto image = row["image"];
@ -109,7 +109,7 @@ TEST_F(MindDataTestPipeline, TestRandomDatasetBasicWithPipeline) {
std::unordered_map<std::string, mindspore::MSTensor> row; std::unordered_map<std::string, mindspore::MSTensor> row;
iter->GetNextRow(&row); iter->GetNextRow(&row);
// Check if RandomDataOp read correct columns // Check if RandomData() read correct columns
uint64_t i = 0; uint64_t i = 0;
while (row.size() != 0) { while (row.size() != 0) {
auto image = row["image"]; auto image = row["image"];
@ -165,7 +165,7 @@ TEST_F(MindDataTestPipeline, TestRandomDatasetBasic2) {
std::unordered_map<std::string, mindspore::MSTensor> row; std::unordered_map<std::string, mindspore::MSTensor> row;
iter->GetNextRow(&row); iter->GetNextRow(&row);
// Check if RandomDataOp read correct columns // Check if RandomData() read correct columns
uint64_t i = 0; uint64_t i = 0;
while (row.size() != 0) { while (row.size() != 0) {
// If no schema specified, RandomData will generate random columns // If no schema specified, RandomData will generate random columns
@ -210,7 +210,7 @@ TEST_F(MindDataTestPipeline, TestRandomDatasetBasic3) {
std::vector<int64_t> expect_2d = {2, 2}; std::vector<int64_t> expect_2d = {2, 2};
std::vector<int64_t> expect_3d = {2, 2, 2}; std::vector<int64_t> expect_3d = {2, 2, 2};
// Check if RandomDataOp read correct columns // Check if RandomData() read correct columns
uint64_t i = 0; uint64_t i = 0;
while (row.size() != 0) { while (row.size() != 0) {
auto col_sint16 = row["col_sint16"]; auto col_sint16 = row["col_sint16"];
@ -292,7 +292,7 @@ TEST_F(MindDataTestPipeline, TestRandomDatasetBasic4) {
std::vector<int64_t> expect_2d = {2, 2}; std::vector<int64_t> expect_2d = {2, 2};
std::vector<int64_t> expect_3d = {2, 2, 2}; std::vector<int64_t> expect_3d = {2, 2, 2};
// Check if RandomDataOp read correct columns // Check if RandomData() read correct columns
uint64_t i = 0; uint64_t i = 0;
while (row.size() != 0) { while (row.size() != 0) {
auto col_sint16 = row["col_sint16"]; auto col_sint16 = row["col_sint16"];
@ -372,7 +372,7 @@ TEST_F(MindDataTestPipeline, TestRandomDatasetBasic5) {
std::vector<int64_t> expect_num = {1}; std::vector<int64_t> expect_num = {1};
std::vector<int64_t> expect_1d = {2}; std::vector<int64_t> expect_1d = {2};
// Check if RandomDataOp read correct columns // Check if RandomData() read correct columns
uint64_t i = 0; uint64_t i = 0;
while (row.size() != 0) { while (row.size() != 0) {
EXPECT_EQ(row.size(), 3); EXPECT_EQ(row.size(), 3);
@ -427,7 +427,7 @@ TEST_F(MindDataTestPipeline, TestRandomDatasetBasic6) {
std::unordered_map<std::string, mindspore::MSTensor> row; std::unordered_map<std::string, mindspore::MSTensor> row;
iter->GetNextRow(&row); iter->GetNextRow(&row);
// Check if RandomDataOp read correct columns // Check if RandomData() read correct columns
uint64_t i = 0; uint64_t i = 0;
while (row.size() != 0) { while (row.size() != 0) {
iter->GetNextRow(&row); iter->GetNextRow(&row);
@ -461,7 +461,7 @@ TEST_F(MindDataTestPipeline, TestRandomDatasetBasic7) {
std::unordered_map<std::string, mindspore::MSTensor> row; std::unordered_map<std::string, mindspore::MSTensor> row;
iter->GetNextRow(&row); iter->GetNextRow(&row);
// Check if RandomDataOp read correct columns // Check if RandomData() read correct columns
uint64_t i = 0; uint64_t i = 0;
while (row.size() != 0) { while (row.size() != 0) {
iter->GetNextRow(&row); iter->GetNextRow(&row);

52
tests/ut/cpp/dataset/c_api_dataset_voc_test.cc
Unescape View File

@ -15,6 +15,7 @@
*/ */
#include "common/common.h" #include "common/common.h"
#include "minddata/dataset/include/datasets.h" #include "minddata/dataset/include/datasets.h"
#include "minddata/dataset/core/tensor.h"
using namespace mindspore::dataset; using namespace mindspore::dataset;
using mindspore::dataset::DataType; using mindspore::dataset::DataType;
@ -47,20 +48,22 @@ TEST_F(MindDataTestPipeline, TestVOCClassIndex) {
std::unordered_map<std::string, mindspore::MSTensor> row; std::unordered_map<std::string, mindspore::MSTensor> row;
iter->GetNextRow(&row); iter->GetNextRow(&row);
// Check if VOCOp read correct labels // Check if VOC() read correct labels
// When we provide class_index, label of ["car","cat","train"] become [0,1,9] // When we provide class_index, label of ["car","cat","train"] become [0,1,9]
// std::shared_ptr<Tensor> expect_label; std::shared_ptr<Tensor> de_expect_label;
// Tensor::CreateFromMemory(TensorShape({1, 1}), DataType(DataType::DE_UINT32), nullptr, &expect_label); ASSERT_OK(Tensor::CreateFromMemory(TensorShape({1, 1}), DataType(DataType::DE_UINT32), nullptr, &de_expect_label));
uint32_t expect[] = {9, 9, 9, 1, 1, 0};
// uint32_t expect[] = {9, 9, 9, 1, 1, 0};
uint64_t i = 0; uint64_t i = 0;
while (row.size() != 0) { while (row.size() != 0) {
auto image = row["image"]; auto image = row["image"];
auto label = row["label"]; auto label = row["label"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape(); MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
MS_LOG(INFO) << "Tensor label shape: " << label.Shape(); MS_LOG(INFO) << "Tensor label shape: " << label.Shape();
// expect_label->SetItemAt({0, 0}, expect[i]);
// EXPECT_EQ(*label, *expect_label); ASSERT_OK(de_expect_label->SetItemAt({0, 0}, expect[i]));
mindspore::MSTensor expect_label = mindspore::MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_expect_label));
EXPECT_MSTENSOR_EQ(label, expect_label);
iter->GetNextRow(&row); iter->GetNextRow(&row);
i++; i++;
@ -132,9 +135,13 @@ TEST_F(MindDataTestPipeline, TestVOCDetection) {
std::unordered_map<std::string, mindspore::MSTensor> row; std::unordered_map<std::string, mindspore::MSTensor> row;
iter->GetNextRow(&row); iter->GetNextRow(&row);
// Check if VOCOp read correct images/labels // Check if VOC() read correct images/labels
// std::string expect_file[] = {"15", "32", "33", "39"}; std::string expect_file[] = {"15", "32", "33", "39"};
// uint32_t expect_num[] = {5, 5, 4, 3}; uint32_t expect_num[] = {5, 5, 4, 3};
std::shared_ptr<Tensor> de_expect_label;
ASSERT_OK(Tensor::CreateFromMemory(TensorShape({1, 1}), DataType(DataType::DE_UINT32), nullptr, &de_expect_label));
uint64_t i = 0; uint64_t i = 0;
while (row.size() != 0) { while (row.size() != 0) {
auto image = row["image"]; auto image = row["image"];
@ -142,14 +149,12 @@ TEST_F(MindDataTestPipeline, TestVOCDetection) {
MS_LOG(INFO) << "Tensor image shape: " << image.Shape(); MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
MS_LOG(INFO) << "Tensor label shape: " << label.Shape(); MS_LOG(INFO) << "Tensor label shape: " << label.Shape();
// std::shared_ptr<Tensor> expect_image; mindspore::MSTensor expect_image = ReadFileToTensor(folder_path + "/JPEGImages/" + expect_file[i] + ".jpg");
// Tensor::CreateFromFile(folder_path + "/JPEGImages/" + expect_file[i] + ".jpg", &expect_image); EXPECT_MSTENSOR_EQ(image, expect_image);
// EXPECT_EQ(*image, *expect_image);
// std::shared_ptr<Tensor> expect_label; ASSERT_OK(de_expect_label->SetItemAt({0, 0}, expect_num[i]));
// Tensor::CreateFromMemory(TensorShape({1, 1}), DataType(DataType::DE_UINT32), nullptr, &expect_label); mindspore::MSTensor expect_label = mindspore::MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_expect_label));
// expect_label->SetItemAt({0, 0}, expect_num[i]); EXPECT_MSTENSOR_EQ(label, expect_label);
// EXPECT_EQ(*label, *expect_label);
iter->GetNextRow(&row); iter->GetNextRow(&row);
i++; i++;
@ -205,9 +210,8 @@ TEST_F(MindDataTestPipeline, TestVOCSegmentation) {
std::unordered_map<std::string, mindspore::MSTensor> row; std::unordered_map<std::string, mindspore::MSTensor> row;
iter->GetNextRow(&row); iter->GetNextRow(&row);
// Check if VOCOp read correct images/targets // Check if VOC() read correct images/targets
// using Tensor = mindspore::dataset::Tensor; std::string expect_file[] = {"32", "33", "39", "32", "33", "39"};
// std::string expect_file[] = {"32", "33", "39", "32", "33", "39"};
uint64_t i = 0; uint64_t i = 0;
while (row.size() != 0) { while (row.size() != 0) {
auto image = row["image"]; auto image = row["image"];
@ -215,13 +219,11 @@ TEST_F(MindDataTestPipeline, TestVOCSegmentation) {
MS_LOG(INFO) << "Tensor image shape: " << image.Shape(); MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
MS_LOG(INFO) << "Tensor target shape: " << target.Shape(); MS_LOG(INFO) << "Tensor target shape: " << target.Shape();
// std::shared_ptr<Tensor> expect_image; mindspore::MSTensor expect_image = ReadFileToTensor(folder_path + "/JPEGImages/" + expect_file[i] + ".jpg");
// Tensor::CreateFromFile(folder_path + "/JPEGImages/" + expect_file[i] + ".jpg", &expect_image); EXPECT_MSTENSOR_EQ(image, expect_image);
// EXPECT_EQ(*image, *expect_image);
// std::shared_ptr<Tensor> expect_target; mindspore::MSTensor expect_target = ReadFileToTensor(folder_path + "/SegmentationClass/" + expect_file[i] + ".png");
// Tensor::CreateFromFile(folder_path + "/SegmentationClass/" + expect_file[i] + ".png", &expect_target); EXPECT_MSTENSOR_EQ(target, expect_target);
// EXPECT_EQ(*target, *expect_target);
iter->GetNextRow(&row); iter->GetNextRow(&row);
i++; i++;

34
tests/ut/cpp/dataset/c_api_datasets_test.cc
Unescape View File

@ -15,6 +15,7 @@
*/ */
#include "common/common.h" #include "common/common.h"
#include "minddata/dataset/include/datasets.h" #include "minddata/dataset/include/datasets.h"
#include "minddata/dataset/core/tensor.h"
using namespace mindspore::dataset; using namespace mindspore::dataset;
using mindspore::dataset::Tensor; using mindspore::dataset::Tensor;
@ -43,25 +44,26 @@ TEST_F(MindDataTestPipeline, TestCelebADataset) {
std::unordered_map<std::string, mindspore::MSTensor> row; std::unordered_map<std::string, mindspore::MSTensor> row;
iter->GetNextRow(&row); iter->GetNextRow(&row);
// Check if CelebAOp read correct images/attr // Check if CelebA() read correct images/attr
// std::string expect_file[] = {"1.JPEG", "2.jpg"}; std::string expect_file[] = {"1.JPEG", "2.jpg"};
// std::vector<std::vector<uint32_t>> expect_attr_vector = { std::vector<std::vector<uint32_t>> expect_attr_vector = {
// {0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, {0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1,
// 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 1, 0, 0, 1}, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 1, 0, 0, 1},
// {0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, {0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1,
// 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1}}; 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1}};
uint64_t i = 0; uint64_t i = 0;
while (row.size() != 0) { while (row.size() != 0) {
// auto image = row["image"]; auto image = row["image"];
// auto attr = row["attr"]; auto attr = row["attr"];
// std::shared_ptr<Tensor> expect_image; mindspore::MSTensor expect_image = ReadFileToTensor(folder_path + expect_file[i]);
// Tensor::CreateFromFile(folder_path + expect_file[i], &expect_image); EXPECT_MSTENSOR_EQ(image, expect_image);
// EXPECT_EQ(*image, *expect_image);
// std::shared_ptr<Tensor> expect_attr; std::shared_ptr<Tensor> de_expect_attr;
// Tensor::CreateFromVector(expect_attr_vector[i], TensorShape({40}), &expect_attr); ASSERT_OK(Tensor::CreateFromVector(expect_attr_vector[i], TensorShape({40}), &de_expect_attr));
// EXPECT_EQ(*attr, *expect_attr); mindspore::MSTensor expect_attr =
mindspore::MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_expect_attr));
EXPECT_MSTENSOR_EQ(attr, expect_attr);
iter->GetNextRow(&row); iter->GetNextRow(&row);
i++; i++;
@ -90,7 +92,7 @@ TEST_F(MindDataTestPipeline, TestCelebADefault) {
std::unordered_map<std::string, mindspore::MSTensor> row; std::unordered_map<std::string, mindspore::MSTensor> row;
iter->GetNextRow(&row); iter->GetNextRow(&row);
// Check if CelebAOp read correct images/attr // Check if CelebA() read correct images/attr
uint64_t i = 0; uint64_t i = 0;
while (row.size() != 0) { while (row.size() != 0) {
auto image = row["image"]; auto image = row["image"];

4
tests/ut/cpp/dataset/c_api_vision_soft_dvpp_test.cc
Unescape View File

@ -142,8 +142,8 @@ TEST_F(MindDataTestPipeline, TestSoftDvppDecodeResizeJpegSuccess1) {
uint64_t i = 0; uint64_t i = 0;
while (row.size() != 0) { while (row.size() != 0) {
i++; i++;
// std::shared_ptr<TensorTransform> image = row["image"]; auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape(); MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row); iter->GetNextRow(&row);
} }

32
tests/ut/cpp/dataset/common/common.cc
Unescape View File

@ -14,6 +14,10 @@
* limitations under the License. * limitations under the License.
*/ */
#include "common.h" #include "common.h"
#include <fstream>
#include <algorithm>
#include <string>
#include <vector>
namespace UT { namespace UT {
#ifdef __cplusplus #ifdef __cplusplus
@ -44,6 +48,34 @@ std::vector<mindspore::dataset::DataType> DatasetOpTesting::ToDETypes(const std:
return ret_t; return ret_t;
} }
// Function to read a file into an MSTensor
// Note: This provides the analogous support for DETensor's CreateFromFile.
mindspore::MSTensor DatasetOpTesting::ReadFileToTensor(const std::string &file) {
if (file.empty()) {
MS_LOG(ERROR) << "Pointer file is nullptr; return an empty Tensor.";
return mindspore::MSTensor();
}
std::ifstream ifs(file);
if (!ifs.good()) {
MS_LOG(ERROR) << "File: " << file << " does not exist; return an empty Tensor.";
return mindspore::MSTensor();
}
if (!ifs.is_open()) {
MS_LOG(ERROR) << "File: " << file << " open failed; return an empty Tensor.";
return mindspore::MSTensor();
}
ifs.seekg(0, std::ios::end);
size_t size = ifs.tellg();
mindspore::MSTensor buf("file", mindspore::DataType::kNumberTypeUInt8, {static_cast<int64_t>(size)}, nullptr, size);
ifs.seekg(0, std::ios::beg);
ifs.read(reinterpret_cast<char *>(buf.MutableData()), size);
ifs.close();
return buf;
}
#ifdef __cplusplus #ifdef __cplusplus
#if __cplusplus #if __cplusplus
} }

16
tests/ut/cpp/dataset/common/common.h
Unescape View File

@ -13,8 +13,8 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
#ifndef TESTS_DATASET_UT_CORE_COMMON_DE_UT_COMMON_H_ #ifndef TESTS_UT_CPP_DATASET_COMMON_COMMON_H_
#define TESTS_DATASET_UT_CORE_COMMON_DE_UT_COMMON_H_ #define TESTS_UT_CPP_DATASET_COMMON_COMMON_H_
#include "gtest/gtest.h" #include "gtest/gtest.h"
#include "include/api/status.h" #include "include/api/status.h"
@ -62,6 +62,15 @@ using mindspore::StatusCode;
} \ } \
} while (false) } while (false)
// Macro to compare 2 MSTensors; compare shape-size and data
#define EXPECT_MSTENSOR_EQ(_mstensor1, _mstensor2) \
do { \
EXPECT_EQ(_mstensor1.Shape().size(), _mstensor2.Shape().size()); \
EXPECT_EQ(_mstensor1.DataSize(), _mstensor2.DataSize()); \
EXPECT_EQ(std::memcmp((const void *)_mstensor1.Data().get(), (const void *)_mstensor2.Data().get(), \
_mstensor2.DataSize()), 0); \
} while (false)
namespace UT { namespace UT {
class Common : public testing::Test { class Common : public testing::Test {
public: public:
@ -75,9 +84,10 @@ class DatasetOpTesting : public Common {
public: public:
std::vector<mindspore::dataset::TensorShape> ToTensorShapeVec(const std::vector<std::vector<int64_t>> &v); std::vector<mindspore::dataset::TensorShape> ToTensorShapeVec(const std::vector<std::vector<int64_t>> &v);
std::vector<mindspore::dataset::DataType> ToDETypes(const std::vector<mindspore::DataType> &t); std::vector<mindspore::dataset::DataType> ToDETypes(const std::vector<mindspore::DataType> &t);
mindspore::MSTensor ReadFileToTensor(const std::string &file);
std::string datasets_root_path_; std::string datasets_root_path_;
std::string mindrecord_root_path_; std::string mindrecord_root_path_;
void SetUp() override; void SetUp() override;
}; };
} // namespace UT } // namespace UT
#endif // TESTS_DATASET_UT_CORE_COMMON_DE_UT_COMMON_H_ #endif // TESTS_UT_CPP_DATASET_COMMON_COMMON_H_

33
tests/ut/cpp/dataset/execute_test.cc
Unescape View File

@ -13,9 +13,7 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
#include <fstream>
#include "common/common.h" #include "common/common.h"
#include "common/cvop_common.h"
#include "include/api/types.h" #include "include/api/types.h"
#include "minddata/dataset/core/de_tensor.h" #include "minddata/dataset/core/de_tensor.h"
#include "minddata/dataset/include/execute.h" #include "minddata/dataset/include/execute.h"
@ -28,39 +26,10 @@ using mindspore::LogStream;
using mindspore::ExceptionType::NoExceptionType; using mindspore::ExceptionType::NoExceptionType;
using mindspore::MsLogLevel::INFO; using mindspore::MsLogLevel::INFO;
class MindDataTestExecute : public UT::CVOP::CVOpCommon { class MindDataTestExecute : public UT::DatasetOpTesting {
protected: protected:
MindDataTestExecute() : CVOpCommon() {}
std::shared_ptr<Tensor> output_tensor_;
}; };
mindspore::MSTensor ReadFileToTensor(const std::string &file) {
if (file.empty()) {
MS_LOG(ERROR) << "Pointer file is nullptr, return an empty Tensor.";
return mindspore::MSTensor();
}
std::ifstream ifs(file);
if (!ifs.good()) {
MS_LOG(ERROR) << "File: " << file << " does not exist, return an empty Tensor.";
return mindspore::MSTensor();
}
if (!ifs.is_open()) {
MS_LOG(ERROR) << "File: " << file << "open failed, return an empty Tensor.";
return mindspore::MSTensor();
}
ifs.seekg(0, std::ios::end);
size_t size = ifs.tellg();
mindspore::MSTensor buf("file", mindspore::DataType::kNumberTypeUInt8, {static_cast<int64_t>(size)}, nullptr, size);
ifs.seekg(0, std::ios::beg);
ifs.read(reinterpret_cast<char *>(buf.MutableData()), size);
ifs.close();
return buf;
}
TEST_F(MindDataTestExecute, TestComposeTransforms) { TEST_F(MindDataTestExecute, TestComposeTransforms) {
MS_LOG(INFO) << "Doing TestComposeTransforms."; MS_LOG(INFO) << "Doing TestComposeTransforms.";

Write Preview
Loading…
Cancel
Save