m53297601
/
mindspore
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

dataset: Reinstate INFO logging and data verification - part 3

Browse Source 
Updates: GetItemAt, CreateFromVector, CreateScalar.
Add TEST_MS_LOG_MSTENSOR.
Wrap ASSERT_OK around Tensor function calls.
pull/13506/head
Cathy Wong 4 years ago
parent c0f41deeae
commit 1775c4e83d
19 changed files with 1138 additions and 796 deletions
Show Stats Download Patch File Download Diff File

170
tests/ut/cpp/dataset/c_api_dataset_clue_test.cc
View File

File diff suppressed because it is too large Load Diff

6
tests/ut/cpp/dataset/c_api_dataset_coco_test.cc
Unescape View File

@ -329,8 +329,7 @@ TEST_F(MindDataTestPipeline, TestCocoPanoptic) {
EXPECT_MSTENSOR_EQ(bbox, expect_bbox);
std::shared_ptr<Tensor> de_expect_categoryid;
ASSERT_OK(
Tensor::CreateFromVector(expect_categoryid_vector[i], TensorShape({bbox_size, 1}), &de_expect_categoryid));
ASSERT_OK(Tensor::CreateFromVector(expect_categoryid_vector[i], TensorShape({bbox_size, 1}), &de_expect_categoryid));
mindspore::MSTensor expect_categoryid =
mindspore::MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_expect_categoryid));
EXPECT_MSTENSOR_EQ(category_id, expect_categoryid);
@ -420,8 +419,7 @@ TEST_F(MindDataTestPipeline, TestCocoStuff) {
EXPECT_MSTENSOR_EQ(image, expect_image);
std::shared_ptr<Tensor> de_expect_segmentation;
ASSERT_OK(
Tensor::CreateFromVector(expect_segmentation_vector[i], TensorShape(expect_size[i]), &de_expect_segmentation));
ASSERT_OK(Tensor::CreateFromVector(expect_segmentation_vector[i], TensorShape(expect_size[i]), &de_expect_segmentation));
mindspore::MSTensor expect_segmentation =
mindspore::MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_expect_segmentation));
EXPECT_MSTENSOR_EQ(segmentation, expect_segmentation);

53
tests/ut/cpp/dataset/c_api_dataset_config_test.cc
Unescape View File

@ -132,18 +132,21 @@ TEST_F(MindDataTestPipeline, TestShuffleWithSeed) {
iter->GetNextRow(&row);
EXPECT_NE(row.find("text"), row.end());
// std::vector<std::string> expected_result = {"Good luck to everyone.", "Be happy every day.", "This is a text
// file."};
std::vector<std::string> expected_result = {"Good luck to everyone.", "Be happy every day.", "This is a text file."};
uint64_t i = 0;
while (row.size() != 0) {
// auto text = row["text"];
// std::string_view sv;
// text->GetItemAt(&sv, {0});
// std::string ss(sv);
// MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50);
auto text = row["text"];
std::shared_ptr<Tensor> de_text;
ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text));
std::string_view sv;
de_text->GetItemAt(&sv, {0});
std::string ss(sv);
MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50);
// Compare against expected result
// EXPECT_STREQ(ss.c_str(), expected_result[i].c_str());
EXPECT_STREQ(ss.c_str(), expected_result[i].c_str());
i++;
iter->GetNextRow(&row);
}
@ -194,22 +197,26 @@ TEST_F(MindDataTestPipeline, TestCallShuffleTwice) {
iter->GetNextRow(&row);
EXPECT_NE(row.find("text"), row.end());
// std::vector<std::string> first_copy;
// std::vector<std::string> second_copy;
std::vector<std::string> first_copy;
std::vector<std::string> second_copy;
uint64_t i = 0;
while (row.size() != 0) {
// auto text = row["text"];
// std::string_view sv;
// text->GetItemAt(&sv, {0});
// std::string ss(sv);
// MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50);
auto text = row["text"];
std::shared_ptr<Tensor> de_text;
ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text));
std::string_view sv;
de_text->GetItemAt(&sv, {0});
std::string ss(sv);
MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50);
// The first three samples are the first copy and the rest are the second
// if (i < 3) {
// first_copy.push_back(ss);
// } else {
// second_copy.push_back(ss);
// }
if (i < 3) {
first_copy.push_back(ss);
} else {
second_copy.push_back(ss);
}
i++;
iter->GetNextRow(&row);
}
@ -218,9 +225,9 @@ TEST_F(MindDataTestPipeline, TestCallShuffleTwice) {
EXPECT_EQ(i, 6);
// Compare the two copies which are deterministic difference
// for (int j = 0; j < 3; j++) {
// EXPECT_STRNE(first_copy.at(j).c_str(), second_copy.at(j).c_str());
// }
for (int j = 0; j < 3; j++) {
EXPECT_STRNE(first_copy.at(j).c_str(), second_copy.at(j).c_str());
}
// Manually terminate the pipeline
iter->Stop();

244
tests/ut/cpp/dataset/c_api_dataset_csv_test.cc
View File

File diff suppressed because it is too large Load Diff

15
tests/ut/cpp/dataset/c_api_dataset_manifest_test.cc
Unescape View File

@ -238,15 +238,20 @@ TEST_F(MindDataTestPipeline, TestManifestClassIndex) {
iter->GetNextRow(&row);
uint64_t i = 0;
// int32_t label_idx = 0;
int32_t label_idx = 0;
while (row.size() != 0) {
i++;
auto image = row["image"];
auto label = row["label"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
// row["label"]->GetItemAt<int32_t>(&label_idx, {});
// MS_LOG(INFO) << "Tensor label value: " << label_idx;
// auto label_it = std::find(expected_label.begin(), expected_label.end(), label_idx);
// EXPECT_NE(label_it, expected_label.end());
std::shared_ptr<Tensor> de_label;
ASSERT_OK(Tensor::CreateFromMSTensor(label, &de_label));
de_label->GetItemAt<int32_t>(&label_idx, {});
MS_LOG(INFO) << "Tensor label value: " << label_idx;
auto label_it = std::find(expected_label.begin(), expected_label.end(), label_idx);
EXPECT_NE(label_it, expected_label.end());
iter->GetNextRow(&row);
}

35
tests/ut/cpp/dataset/c_api_dataset_minddata_test.cc
Unescape View File

@ -46,8 +46,8 @@ TEST_F(MindDataTestPipeline, TestMindDataSuccess1) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto image = row["file_name"];
// MS_LOG(INFO) << "Tensor image file name: " << *image;
auto image = row["file_name"];
TEST_MS_LOG_MSTENSOR(INFO, "Tensor image file name: ", image);
iter->GetNextRow(&row);
}
@ -96,8 +96,9 @@ TEST_F(MindDataTestPipeline, TestMindDataSuccess2) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto image = row["file_name"];
// MS_LOG(INFO) << "Tensor image file name: " << *image;
auto image = row["file_name"];
TEST_MS_LOG_MSTENSOR(INFO, "Tensor image file name: ", image);
iter->GetNextRow(&row);
}
@ -131,8 +132,9 @@ TEST_F(MindDataTestPipeline, TestMindDataSuccess3) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto image = row["file_name"];
// MS_LOG(INFO) << "Tensor image file name: " << *image;
auto image = row["file_name"];
TEST_MS_LOG_MSTENSOR(INFO, "Tensor image file name: ", image);
iter->GetNextRow(&row);
}
@ -165,8 +167,8 @@ TEST_F(MindDataTestPipeline, TestMindDataSuccess4) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto label = row["label"];
// MS_LOG(INFO) << "Tensor label: " << *label;
auto label = row["label"];
TEST_MS_LOG_MSTENSOR(INFO, "Tensor label: ", label);
iter->GetNextRow(&row);
}
@ -266,7 +268,8 @@ TEST_F(MindDataTestPipeline, TestMindDataSuccess6) {
uint64_t j = 0;
while (row.size() != 0) {
j++;
// MS_LOG(INFO) << "Tensor label: " << *row["label"];
auto label = row["label"];
TEST_MS_LOG_MSTENSOR(INFO, "Tensor label: ", label);
iter->GetNextRow(&row);
}
EXPECT_EQ(j, expected_samples[i]);
@ -312,10 +315,10 @@ TEST_F(MindDataTestPipeline, TestMindDataSuccess7) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto image = row["file_name"];
auto image = row["file_name"];
auto label = row["label"];
// MS_LOG(INFO) << "Tensor file name: " << *image;
// MS_LOG(INFO) << "Tensor label: " << *label;
TEST_MS_LOG_MSTENSOR(INFO, "Tensor image file name: ", image);
TEST_MS_LOG_MSTENSOR(INFO, "Tensor label: ", label);
EXPECT_MSTENSOR_EQ(label, expect_item);
@ -383,10 +386,10 @@ TEST_F(MindDataTestPipeline, TestMindDataSuccess8) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto image = row["file_name"];
auto image = row["file_name"];
auto label = row["label"];
// MS_LOG(INFO) << "Tensor file name: " << *image;
// MS_LOG(INFO) << "Tensor label: " << *label;
TEST_MS_LOG_MSTENSOR(INFO, "Tensor image file name: ", image);
TEST_MS_LOG_MSTENSOR(INFO, "Tensor label: ", label);
EXPECT_MSTENSOR_EQ(label, expect_item);
@ -459,7 +462,7 @@ TEST_F(MindDataTestPipeline, TestMindDataSuccess9) {
while (row.size() != 0) {
i++;
auto label = row["label"];
// MS_LOG(INFO) << "Tensor label: " << *label;
TEST_MS_LOG_MSTENSOR(INFO, "Tensor label: ", label);
EXPECT_MSTENSOR_EQ(label, expect_item);

36
tests/ut/cpp/dataset/c_api_dataset_ops_test.cc
Unescape View File

@ -632,20 +632,24 @@ TEST_F(MindDataTestPipeline, TestFilterSuccess1) {
std::unordered_map<std::string, mindspore::MSTensor> row;
iter->GetNextRow(&row);
// std::vector<uint64_t> label_list;
std::vector<uint64_t> label_list;
uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto label = row["label"];
// uint64_t label_value;
// label->GetItemAt(&label_value, {0});
// label_list.push_back(label_value);
auto label = row["label"];
std::shared_ptr<Tensor> de_label;
uint64_t label_value;
ASSERT_OK(Tensor::CreateFromMSTensor(label, &de_label));
de_label->GetItemAt(&label_value, {0});
label_list.push_back(label_value);
iter->GetNextRow(&row);
}
// Only 1 column whose label is equal to 3
EXPECT_EQ(i, 1);
// EXPECT_EQ(label_list.at(0), 3);
EXPECT_EQ(label_list.at(0), 3);
// Manually terminate the pipeline
iter->Stop();
@ -674,21 +678,25 @@ TEST_F(MindDataTestPipeline, TestFilterSuccess2) {
std::unordered_map<std::string, mindspore::MSTensor> row;
iter->GetNextRow(&row);
// std::vector<uint64_t> label_list;
std::vector<uint64_t> label_list;
uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto label = row["label"];
// uint64_t label_value;
// label->GetItemAt(&label_value, {0});
// label_list.push_back(label_value);
auto label = row["label"];
std::shared_ptr<Tensor> de_label;
uint64_t label_value;
ASSERT_OK(Tensor::CreateFromMSTensor(label, &de_label));
de_label->GetItemAt(&label_value, {0});
label_list.push_back(label_value);
iter->GetNextRow(&row);
}
// There are 2 columns whose label is more than 1
EXPECT_EQ(i, 2);
// EXPECT_EQ(label_list.at(0), 2);
// EXPECT_EQ(label_list.at(1), 3);
EXPECT_EQ(label_list.at(0), 2);
EXPECT_EQ(label_list.at(1), 3);
// Manually terminate the pipeline
iter->Stop();
@ -1079,7 +1087,7 @@ TEST_F(MindDataTestPipeline, TestProjectMapAutoInjection) {
i++;
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
// EXPECT_EQ(image.Shape()[0], 30);
EXPECT_EQ(image.Shape()[0], 30);
iter->GetNextRow(&row);
}

14
tests/ut/cpp/dataset/c_api_dataset_save.cc
Unescape View File

@ -46,9 +46,10 @@ TEST_F(MindDataTestPipeline, TestSaveCifar10AndLoad) {
// Save original data for comparison
uint64_t i = 0;
while (row.size() != 0) {
// auto label = row["label"];
// original_data.push_back(label);
// MS_LOG(INFO) << "Tensor label: " << *label;
auto label = row["label"];
original_data.push_back(label);
TEST_MS_LOG_MSTENSOR(INFO, "Tensor label: ", label);
iter->GetNextRow(&row);
i++;
}
@ -97,9 +98,10 @@ TEST_F(MindDataTestPipeline, TestSaveCifar10AndLoad) {
// Expect the output data is same with original_data
uint64_t j = 0;
while (row_minddata.size() != 0) {
// auto label = row_minddata["label"];
// EXPECT_EQ(*original_data[j], *label);
// MS_LOG(INFO) << "Tensor label: " << *label;
auto label = row_minddata["label"];
EXPECT_MSTENSOR_EQ(original_data[j], label);
TEST_MS_LOG_MSTENSOR(INFO, "Tensor label: ", label);
iter_minddata->GetNextRow(&row_minddata);
j++;
}

150
tests/ut/cpp/dataset/c_api_dataset_textfile_test.cc
Unescape View File

@ -60,12 +60,15 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetBasic) {
while (row.size() != 0) {
auto text = row["text"];
MS_LOG(INFO) << "Tensor text shape: " << text.Shape();
// std::string_view sv;
// text->GetItemAt(&sv, {0});
// std::string ss(sv);
// MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50);
// // Compare against expected result
// EXPECT_STREQ(ss.c_str(), expected_result[i].c_str());
std::shared_ptr<Tensor> de_text;
ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text));
std::string_view sv;
de_text->GetItemAt(&sv, {0});
std::string ss(sv);
MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50);
// Compare against expected result
EXPECT_STREQ(ss.c_str(), expected_result[i].c_str());
i++;
iter->GetNextRow(&row);
}
@ -317,12 +320,15 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleFalse1A) {
while (row.size() != 0) {
auto text = row["text"];
MS_LOG(INFO) << "Tensor text shape: " << text.Shape();
// std::string_view sv;
// text->GetItemAt(&sv, {0});
// std::string ss(sv);
// MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50);
// // Compare against expected result
// EXPECT_STREQ(ss.c_str(), expected_result[i].c_str());
std::shared_ptr<Tensor> de_text;
ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text));
std::string_view sv;
de_text->GetItemAt(&sv, {0});
std::string ss(sv);
MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50);
// Compare against expected result
EXPECT_STREQ(ss.c_str(), expected_result[i].c_str());
i++;
iter->GetNextRow(&row);
}
@ -375,12 +381,15 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleFalse1B) {
while (row.size() != 0) {
auto text = row["text"];
MS_LOG(INFO) << "Tensor text shape: " << text.Shape();
// std::string_view sv;
// text->GetItemAt(&sv, {0});
// std::string ss(sv);
// MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50);
// // Compare against expected result
// EXPECT_STREQ(ss.c_str(), expected_result[i].c_str());
std::shared_ptr<Tensor> de_text;
ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text));
std::string_view sv;
de_text->GetItemAt(&sv, {0});
std::string ss(sv);
MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50);
// Compare against expected result
EXPECT_STREQ(ss.c_str(), expected_result[i].c_str());
i++;
iter->GetNextRow(&row);
}
@ -432,12 +441,15 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleFalse4Shard) {
while (row.size() != 0) {
auto text = row["text"];
MS_LOG(INFO) << "Tensor text shape: " << text.Shape();
// std::string_view sv;
// text->GetItemAt(&sv, {0});
// std::string ss(sv);
// MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50);
// // Compare against expected result
// EXPECT_STREQ(ss.c_str(), expected_result[i].c_str());
std::shared_ptr<Tensor> de_text;
ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text));
std::string_view sv;
de_text->GetItemAt(&sv, {0});
std::string ss(sv);
MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50);
// Compare against expected result
EXPECT_STREQ(ss.c_str(), expected_result[i].c_str());
i++;
iter->GetNextRow(&row);
}
@ -492,12 +504,15 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleFiles1A) {
while (row.size() != 0) {
auto text = row["text"];
MS_LOG(INFO) << "Tensor text shape: " << text.Shape();
// std::string_view sv;
// text->GetItemAt(&sv, {0});
// std::string ss(sv);
// MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50);
// // Compare against expected result
// EXPECT_STREQ(ss.c_str(), expected_result[i].c_str());
std::shared_ptr<Tensor> de_text;
ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text));
std::string_view sv;
de_text->GetItemAt(&sv, {0});
std::string ss(sv);
MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50);
// Compare against expected result
EXPECT_STREQ(ss.c_str(), expected_result[i].c_str());
i++;
iter->GetNextRow(&row);
}
@ -552,12 +567,15 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleFiles1B) {
while (row.size() != 0) {
auto text = row["text"];
MS_LOG(INFO) << "Tensor text shape: " << text.Shape();
// std::string_view sv;
// text->GetItemAt(&sv, {0});
// std::string ss(sv);
// MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50);
// // Compare against expected result
// EXPECT_STREQ(ss.c_str(), expected_result[i].c_str());
std::shared_ptr<Tensor> de_text;
ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text));
std::string_view sv;
de_text->GetItemAt(&sv, {0});
std::string ss(sv);
MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50);
// Compare against expected result
EXPECT_STREQ(ss.c_str(), expected_result[i].c_str());
i++;
iter->GetNextRow(&row);
}
@ -611,12 +629,15 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleFiles4) {
while (row.size() != 0) {
auto text = row["text"];
MS_LOG(INFO) << "Tensor text shape: " << text.Shape();
// std::string_view sv;
// text->GetItemAt(&sv, {0});
// std::string ss(sv);
// MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50);
// // Compare against expected result
// EXPECT_STREQ(ss.c_str(), expected_result[i].c_str());
std::shared_ptr<Tensor> de_text;
ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text));
std::string_view sv;
de_text->GetItemAt(&sv, {0});
std::string ss(sv);
MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50);
// Compare against expected result
EXPECT_STREQ(ss.c_str(), expected_result[i].c_str());
i++;
iter->GetNextRow(&row);
}
@ -666,12 +687,15 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleGlobal1A) {
while (row.size() != 0) {
auto text = row["text"];
MS_LOG(INFO) << "Tensor text shape: " << text.Shape();
// std::string_view sv;
// text->GetItemAt(&sv, {0});
// std::string ss(sv);
// MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50);
// // Compare against expected result
// EXPECT_STREQ(ss.c_str(), expected_result[i].c_str());
std::shared_ptr<Tensor> de_text;
ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text));
std::string_view sv;
de_text->GetItemAt(&sv, {0});
std::string ss(sv);
MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50);
// Compare against expected result
EXPECT_STREQ(ss.c_str(), expected_result[i].c_str());
i++;
iter->GetNextRow(&row);
}
@ -724,12 +748,15 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleGlobal1B) {
while (row.size() != 0) {
auto text = row["text"];
MS_LOG(INFO) << "Tensor text shape: " << text.Shape();
// std::string_view sv;
// text->GetItemAt(&sv, {0});
// std::string ss(sv);
// MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50);
// // Compare against expected result
// EXPECT_STREQ(ss.c_str(), expected_result[i].c_str());
std::shared_ptr<Tensor> de_text;
ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text));
std::string_view sv;
de_text->GetItemAt(&sv, {0});
std::string ss(sv);
MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50);
// Compare against expected result
EXPECT_STREQ(ss.c_str(), expected_result[i].c_str());
i++;
iter->GetNextRow(&row);
}
@ -782,12 +809,15 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleGlobal4) {
while (row.size() != 0) {
auto text = row["text"];
MS_LOG(INFO) << "Tensor text shape: " << text.Shape();
// std::string_view sv;
// text->GetItemAt(&sv, {0});
// std::string ss(sv);
// MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50);
// // Compare against expected result
// EXPECT_STREQ(ss.c_str(), expected_result[i].c_str());
std::shared_ptr<Tensor> de_text;
ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text));
std::string_view sv;
de_text->GetItemAt(&sv, {0});
std::string ss(sv);
MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50);
// Compare against expected result
EXPECT_STREQ(ss.c_str(), expected_result[i].c_str());
i++;
iter->GetNextRow(&row);
}

35
tests/ut/cpp/dataset/c_api_dataset_tfrecord_test.cc
Unescape View File

@ -148,12 +148,21 @@ TEST_F(MindDataTestPipeline, TestTFRecordDatasetShuffle) {
iter2->GetNextRow(&row2);
uint64_t i = 0;
// int64_t value1 = 0;
// int64_t value2 = 0;
int64_t value1 = 0;
int64_t value2 = 0;
while (row1.size() != 0 && row2.size() != 0) {
// row1["scalars"]->GetItemAt(&value1, {0});
// row2["scalars"]->GetItemAt(&value2, {0});
// EXPECT_EQ(value1, value2);
auto scalars1 = row1["scalars"];
std::shared_ptr<Tensor> de_scalars1;
ASSERT_OK(Tensor::CreateFromMSTensor(scalars1, &de_scalars1));
de_scalars1->GetItemAt(&value1, {0});
auto scalars2 = row2["scalars"];
std::shared_ptr<Tensor> de_scalars2;
ASSERT_OK(Tensor::CreateFromMSTensor(scalars2, &de_scalars2));
de_scalars2->GetItemAt(&value2, {0});
EXPECT_EQ(value1, value2);
iter1->GetNextRow(&row1);
iter2->GetNextRow(&row2);
i++;
@ -191,17 +200,21 @@ TEST_F(MindDataTestPipeline, TestTFRecordDatasetShuffle2) {
std::unordered_map<std::string, mindspore::MSTensor> row;
iter->GetNextRow(&row);
// std::vector<int> expect = {9, 3, 4, 7, 2, 1, 6, 8, 10, 5};
// std::vector<int> actual = {};
// int64_t value = 0;
std::vector<int> expect = {9, 3, 4, 7, 2, 1, 6, 8, 10, 5};
std::vector<int> actual = {};
int64_t value = 0;
uint64_t i = 0;
while (row.size() != 0) {
// row["scalars"]->GetItemAt(&value, {});
// actual.push_back(value);
auto scalars = row["scalars"];
std::shared_ptr<Tensor> de_scalars;
ASSERT_OK(Tensor::CreateFromMSTensor(scalars, &de_scalars));
de_scalars->GetItemAt(&value, {0});
actual.push_back(value);
iter->GetNextRow(&row);
i++;
}
// ASSERT_EQ(actual, expect);
ASSERT_EQ(actual, expect);
EXPECT_EQ(i, 10);
// Manually terminate the pipeline
iter->Stop();

2
tests/ut/cpp/dataset/c_api_datasets_test.cc
Unescape View File

@ -222,7 +222,7 @@ TEST_F(MindDataTestPipeline, TestImageFolderFailWithWrongExtensionFail) {
std::unordered_map<std::string, mindspore::MSTensor> row;
iter->GetNextRow(&row);
// Expect no data: cannot find files with specified extension
// EXPECT_EQ(row.size(), 0);
EXPECT_EQ(row.size(), 0);
// Manually terminate the pipeline
iter->Stop();

52
tests/ut/cpp/dataset/c_api_epoch_ctrl_test.cc
Unescape View File

@ -26,10 +26,10 @@ class MindDataTestEpochCtrl : public UT::DatasetOpTesting {
TEST_F(MindDataTestEpochCtrl, TestAutoInjectEpoch) {
MS_LOG(INFO) << "Doing MindDataTestEpochCtrl-TestAutoInjectEpoch.";
// int32_t img_class[4] = {0, 1, 2, 3};
int32_t img_class[4] = {0, 1, 2, 3};
int32_t num_epochs = 2 + std::rand() % 3;
int32_t sampler_size = 44;
// int32_t class_size = 11;
int32_t class_size = 11;
MS_LOG(INFO) << "num_epochs: " << num_epochs;
// Create an ImageFolder Dataset
@ -50,10 +50,12 @@ TEST_F(MindDataTestEpochCtrl, TestAutoInjectEpoch) {
iter->GetNextRow(&row);
while (row.size() != 0) {
// auto label = row["label"];
// int32_t label_value;
// label->GetItemAt(&label_value, {0});
// EXPECT_TRUE(img_class[(i % sampler_size) / class_size] == label_value);
auto label = row["label"];
std::shared_ptr<Tensor> de_label;
int64_t label_value;
ASSERT_OK(Tensor::CreateFromMSTensor(label, &de_label));
de_label->GetItemAt(&label_value, {0});
EXPECT_TRUE(img_class[(i % sampler_size) / class_size] == label_value);
iter->GetNextRow(&row);
i++;
@ -64,7 +66,7 @@ TEST_F(MindDataTestEpochCtrl, TestAutoInjectEpoch) {
// Try to fetch data beyond the specified number of epochs.
iter->GetNextRow(&row);
// EXPECT_EQ(row.size(), 2);
EXPECT_EQ(row.size(), 2);
// Manually terminate the pipeline
iter->Stop();
@ -94,10 +96,12 @@ TEST_F(MindDataTestEpochCtrl, TestEpoch) {
for (int epoch = 0; epoch < num_epochs; epoch++) {
iter->GetNextRow(&row);
while (row.size() != 0) {
// auto label = row["label"];
// int32_t label_value;
// label->GetItemAt(&label_value, {0});
// EXPECT_TRUE(label_value >= 0 && label_value <= 3);
auto label = row["label"];
std::shared_ptr<Tensor> de_label;
int64_t label_value;
ASSERT_OK(Tensor::CreateFromMSTensor(label, &de_label));
de_label->GetItemAt(&label_value, {0});
EXPECT_TRUE(label_value >= 0 && label_value <= 3);
iter->GetNextRow(&row);
i++;
@ -109,7 +113,7 @@ TEST_F(MindDataTestEpochCtrl, TestEpoch) {
// Try to fetch data beyond the specified number of epochs.
iter->GetNextRow(&row);
// EXPECT_EQ(row.size(), 2);
EXPECT_EQ(row.size(), 2);
// Manually terminate the pipeline
iter->Stop();
@ -141,10 +145,12 @@ TEST_F(MindDataTestEpochCtrl, TestRepeatEpoch) {
for (int epoch = 0; epoch < num_epochs; epoch++) {
iter->GetNextRow(&row);
while (row.size() != 0) {
// auto label = row["label"];
// int32_t label_value;
// label->GetItemAt(&label_value, {0});
// EXPECT_TRUE(label_value >= 0 && label_value <= 3);
auto label = row["label"];
std::shared_ptr<Tensor> de_label;
int64_t label_value;
ASSERT_OK(Tensor::CreateFromMSTensor(label, &de_label));
de_label->GetItemAt(&label_value, {0});
EXPECT_TRUE(label_value >= 0 && label_value <= 3);
iter->GetNextRow(&row);
i++;
@ -156,7 +162,7 @@ TEST_F(MindDataTestEpochCtrl, TestRepeatEpoch) {
// Try to fetch data beyond the specified number of epochs.
iter->GetNextRow(&row);
// EXPECT_EQ(row.size(), 2);
EXPECT_EQ(row.size(), 2);
// Manually terminate the pipeline
iter->Stop();
@ -188,10 +194,12 @@ TEST_F(MindDataTestEpochCtrl, TestRepeatRepeatEpoch) {
for (int epoch = 0; epoch < num_epochs; epoch++) {
iter->GetNextRow(&row);
while (row.size() != 0) {
// auto label = row["label"];
// int32_t label_value;
// label->GetItemAt(&label_value, {0});
// EXPECT_TRUE(label_value >= 0 && label_value <= 3);
auto label = row["label"];
std::shared_ptr<Tensor> de_label;
int64_t label_value;
ASSERT_OK(Tensor::CreateFromMSTensor(label, &de_label));
de_label->GetItemAt(&label_value, {0});
EXPECT_TRUE(label_value >= 0 && label_value <= 3);
iter->GetNextRow(&row);
i++;
@ -203,7 +211,7 @@ TEST_F(MindDataTestEpochCtrl, TestRepeatRepeatEpoch) {
// Try to fetch data beyond the specified number of epochs.
iter->GetNextRow(&row);
// EXPECT_EQ(row.size(), 2);
EXPECT_EQ(row.size(), 2);
// Manually terminate the pipeline
iter->Stop();

36
tests/ut/cpp/dataset/c_api_text_sentence_piece_vocab_test.cc
Unescape View File

@ -71,15 +71,19 @@ TEST_F(MindDataTestPipeline, TestSentencePieceVocabSuccess1) {
iter->GetNextRow(&row);
// Expected result after tokenization
// std::vector<std::string> expected = {"▁I", "▁sa", "w", "▁a", "▁girl", "▁with", "▁a", "▁te", "les", "co", "pe",
// "."};
std::vector<std::string> expected = {"▁I", "▁sa", "w", "▁a", "▁girl", "▁with", "▁a", "▁te", "les", "co", "pe", "."};
std::shared_ptr<Tensor> de_expected_tensor;
ASSERT_OK(Tensor::CreateFromVector(expected, &de_expected_tensor));
mindspore::MSTensor expected_tensor =
mindspore::MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_expected_tensor));
uint64_t i = 0;
while (row.size() != 0) {
// auto txt = row["text"];
// MS_LOG(INFO) << *txt;
// mindspore::MSTensor expected_tensor;
// Tensor::CreateFromVector(expected, &expected_tensor);
// EXPECT_EQ(*txt, *expected_tensor);
auto txt = row["text"];
TEST_MS_LOG_MSTENSOR(INFO, "txt: ", txt);
EXPECT_MSTENSOR_EQ(txt, expected_tensor);
iter->GetNextRow(&row);
i++;
}
@ -127,15 +131,19 @@ TEST_F(MindDataTestPipeline, TestSentencePieceVocabSuccess2) {
iter->GetNextRow(&row);
// Expected result after tokenization
// std::vector<std::string> expected = {"▁I", "▁sa", "w", "▁a", "▁girl", "▁with", "▁a", "▁te", "les", "co", "pe",
// "."};
std::vector<std::string> expected = {"▁I", "▁sa", "w", "▁a", "▁girl", "▁with", "▁a", "▁te", "les", "co", "pe", "."};
std::shared_ptr<Tensor> de_expected_tensor;
ASSERT_OK(Tensor::CreateFromVector(expected, &de_expected_tensor));
mindspore::MSTensor expected_tensor =
mindspore::MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_expected_tensor));
uint64_t i = 0;
while (row.size() != 0) {
// auto txt = row["text"];
// MS_LOG(INFO) << *txt;
// mindspore::MSTensor expected_tensor;
// Tensor::CreateFromVector(expected, &expected_tensor);
// EXPECT_EQ(*txt, *expected_tensor);
auto txt = row["text"];
TEST_MS_LOG_MSTENSOR(INFO, "txt: ", txt);
EXPECT_MSTENSOR_EQ(txt, expected_tensor);
iter->GetNextRow(&row);
i++;
}

950
tests/ut/cpp/dataset/c_api_text_test.cc
View File

File diff suppressed because it is too large Load Diff

108
tests/ut/cpp/dataset/c_api_text_vocab_test.cc
Unescape View File

@ -35,6 +35,12 @@ class MindDataTestPipeline : public UT::DatasetOpTesting {
protected:
};
// Macro to compare 2 MSTensors as not equal; compare datasize only
#define EXPECT_MSTENSOR_DATA_NE(_mstensor1, _mstensor2) \
do { \
EXPECT_NE(_mstensor1.DataSize(), _mstensor2.DataSize()); \
} while (false)
TEST_F(MindDataTestPipeline, TestVocabLookupOp) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestVocabLookupOp.";
@ -67,13 +73,17 @@ TEST_F(MindDataTestPipeline, TestVocabLookupOp) {
iter->GetNextRow(&row);
uint64_t i = 0;
// std::vector<int32_t> expected = {2, 1, 4, 5, 6, 7};
std::vector<int32_t> expected = {2, 1, 4, 5, 6, 7};
while (row.size() != 0) {
// auto ind = row["text"];
// MS_LOG(INFO) << ind->shape() << " " << *ind;
// mindspore::MSTensor expected_item;
// Tensor::CreateScalar(expected[i], &expected_item);
// EXPECT_EQ(*ind, *expected_item);
auto ind = row["text"];
MS_LOG(INFO) << ind.Shape();
TEST_MS_LOG_MSTENSOR(INFO, "ind: ", ind);
std::shared_ptr<Tensor> de_expected_item;
ASSERT_OK(Tensor::CreateScalar(expected[i], &de_expected_item));
mindspore::MSTensor ms_expected_item =
mindspore::MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_expected_item));
EXPECT_MSTENSOR_EQ(ind, ms_expected_item);
iter->GetNextRow(&row);
i++;
}
@ -111,13 +121,17 @@ TEST_F(MindDataTestPipeline, TestVocabLookupOpEmptyString) {
iter->GetNextRow(&row);
uint64_t i = 0;
// std::vector<int32_t> expected = {2, 1, 4, 5, 6, 7};
std::vector<int32_t> expected = {2, 1, 4, 5, 6, 7};
while (row.size() != 0) {
// auto ind = row["text"];
// MS_LOG(INFO) << ind->shape() << " " << *ind;
// mindspore::MSTensor expected_item;
// Tensor::CreateScalar(expected[i], &expected_item);
// EXPECT_EQ(*ind, *expected_item);
auto ind = row["text"];
MS_LOG(INFO) << ind.Shape();
TEST_MS_LOG_MSTENSOR(INFO, "ind: ", ind);
std::shared_ptr<Tensor> de_expected_item;
ASSERT_OK(Tensor::CreateScalar(expected[i], &de_expected_item));
mindspore::MSTensor ms_expected_item =
mindspore::MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_expected_item));
EXPECT_MSTENSOR_EQ(ind, ms_expected_item);
iter->GetNextRow(&row);
i++;
}
@ -207,13 +221,17 @@ TEST_F(MindDataTestPipeline, TestVocabFromDataset) {
iter->GetNextRow(&row);
uint64_t i = 0;
// std::vector<int32_t> expected = {4, 5, 3, 6, 7, 2};
std::vector<int32_t> expected = {4, 5, 3, 6, 7, 2};
while (row.size() != 0) {
// auto ind = row["text"];
// MS_LOG(INFO) << ind->shape() << " " << *ind;
// mindspore::MSTensor expected_item;
// Tensor::CreateScalar(expected[i], &expected_item);
// EXPECT_EQ(*ind, *expected_item);
auto ind = row["text"];
MS_LOG(INFO) << ind.Shape();
TEST_MS_LOG_MSTENSOR(INFO, "ind: ", ind);
std::shared_ptr<Tensor> de_expected_item;
ASSERT_OK(Tensor::CreateScalar(expected[i], &de_expected_item));
mindspore::MSTensor ms_expected_item =
mindspore::MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_expected_item));
EXPECT_MSTENSOR_EQ(ind, ms_expected_item);
iter->GetNextRow(&row);
i++;
}
@ -253,16 +271,25 @@ TEST_F(MindDataTestPipeline, TestVocabFromDatasetDefault) {
iter->GetNextRow(&row);
uint64_t i = 0;
// std::vector<int32_t> expected = {2, 3, 1, 4, 5, 0};
// std::vector<int64_t> not_expected = {2, 3, 1, 4, 5, 0};
std::vector<int32_t> expected = {2, 3, 1, 4, 5, 0};
std::vector<int64_t> not_expected = {2, 3, 1, 4, 5, 0};
while (row.size() != 0) {
// auto ind = row["text"];
// MS_LOG(INFO) << ind->shape() << " " << *ind;
// mindspore::MSTensor expected_item, not_expected_item;
// Tensor::CreateScalar(expected[i], &expected_item);
// Tensor::CreateScalar(not_expected[i], &not_expected_item);
// EXPECT_EQ(*ind, *expected_item);
// EXPECT_NE(*ind, *not_expected_item);
auto ind = row["text"];
MS_LOG(INFO) << ind.Shape();
TEST_MS_LOG_MSTENSOR(INFO, "ind: ", ind);
std::shared_ptr<Tensor> de_expected_item;
ASSERT_OK(Tensor::CreateScalar(expected[i], &de_expected_item));
mindspore::MSTensor ms_expected_item =
mindspore::MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_expected_item));
EXPECT_MSTENSOR_EQ(ind, ms_expected_item);
std::shared_ptr<Tensor> de_not_expected_item;
ASSERT_OK(Tensor::CreateScalar(not_expected[i], &de_not_expected_item));
mindspore::MSTensor ms_not_expected_item =
mindspore::MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_not_expected_item));
EXPECT_MSTENSOR_DATA_NE(ind, ms_not_expected_item);
iter->GetNextRow(&row);
i++;
}
@ -361,16 +388,25 @@ TEST_F(MindDataTestPipeline, TestVocabFromDatasetInt64) {
iter->GetNextRow(&row);
uint64_t i = 0;
// std::vector<int64_t> expected = {2, 3, 1, 4, 5, 0};
// std::vector<int8_t> not_expected = {2, 3, 1, 4, 5, 0};
std::vector<int64_t> expected = {2, 3, 1, 4, 5, 0};
std::vector<int8_t> not_expected = {2, 3, 1, 4, 5, 0};
while (row.size() != 0) {
// auto ind = row["text"];
// MS_LOG(INFO) << ind->shape() << " " << *ind;
// mindspore::MSTensor expected_item, not_expected_item;
// Tensor::CreateScalar(expected[i], &expected_item);
// Tensor::CreateScalar(not_expected[i], &not_expected_item);
// EXPECT_EQ(*ind, *expected_item);
// EXPECT_NE(*ind, *not_expected_item);
auto ind = row["text"];
MS_LOG(INFO) << ind.Shape();
TEST_MS_LOG_MSTENSOR(INFO, "ind: ", ind);
std::shared_ptr<Tensor> de_expected_item;
ASSERT_OK(Tensor::CreateScalar(expected[i], &de_expected_item));
mindspore::MSTensor ms_expected_item =
mindspore::MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_expected_item));
EXPECT_MSTENSOR_EQ(ind, ms_expected_item);
std::shared_ptr<Tensor> de_not_expected_item;
ASSERT_OK(Tensor::CreateScalar(not_expected[i], &de_not_expected_item));
mindspore::MSTensor ms_not_expected_item =
mindspore::MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_not_expected_item));
EXPECT_MSTENSOR_DATA_NE(ind, ms_not_expected_item);
iter->GetNextRow(&row);
i++;
}

10
tests/ut/cpp/dataset/c_api_transforms_test.cc
Unescape View File

@ -164,12 +164,10 @@ TEST_F(MindDataTestPipeline, TestDuplicateSuccess) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
// FIXME
// auto image = row["image"];
// auto image_copy = row["image_copy"];
// MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
// auto n = memcmp(&image, &image_copy, image.DataSize());
// EXPECT_EQ(n, 0);
auto image = row["image"];
auto image_copy = row["image_copy"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
EXPECT_MSTENSOR_EQ(image, image_copy);
iter->GetNextRow(&row);
}

2
tests/ut/cpp/dataset/c_api_vision_r_to_z_test.cc
Unescape View File

@ -66,7 +66,7 @@ TEST_F(MindDataTestPipeline, TestRescaleSucess1) {
auto image1 = row1["image"];
// EXPECT_EQ(*image, *image1);
EXPECT_MSTENSOR_EQ(image, image1);
// Manually terminate the pipeline
iter1->Stop();

4
tests/ut/cpp/dataset/common/common.cc
Unescape View File

@ -1,5 +1,5 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
* Copyright 2019-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -14,8 +14,8 @@
* limitations under the License.
*/
#include "common.h"
#include <fstream>
#include <algorithm>
#include <fstream>
#include <string>
#include <vector>

8
tests/ut/cpp/dataset/common/common.h
Unescape View File

@ -71,6 +71,14 @@ do {
_mstensor2.DataSize()), 0); \
} while (false)
// Macro to invoke MS_LOG for MSTensor
#define TEST_MS_LOG_MSTENSOR(_loglevel, _msg, _mstensor) \
do { \
std::shared_ptr<Tensor> _de_tensor; \
ASSERT_OK(Tensor::CreateFromMSTensor(_mstensor, &_de_tensor)); \
MS_LOG(_loglevel) << _msg << *_de_tensor; \
} while (false)
namespace UT {
class Common : public testing::Test {
public:

Write Preview
Loading…
Cancel
Save