m53297601
/
mindspore
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

dataset: Reinstate INFO logging and data verification - part1

Browse Source
pull/12936/head
Cathy Wong 4 years ago
parent 9493094f9f
commit 521c313a3b
15 changed files with 321 additions and 308 deletions
Show Stats Download Patch File Download Diff File

22
tests/ut/cpp/dataset/c_api_dataset_album_test.cc
Unescape View File

@ -45,8 +45,8 @@ TEST_F(MindDataTestPipeline, TestAlbumBasic) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape();
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row);
}
@ -100,8 +100,8 @@ TEST_F(MindDataTestPipeline, TestAlbumBasicWithPipeline) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape();
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row);
}
@ -151,13 +151,11 @@ TEST_F(MindDataTestPipeline, TestAlbumDecode) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
/*
auto image = row["image"];
auto shape = image->shape();
MS_LOG(INFO) << "Tensor image shape size: " << shape.Size();
MS_LOG(INFO) << "Tensor image shape: " << image->shape();
EXPECT_GT(shape.Size(), 1); // Verify decode=true took effect
*/
auto shape = image.Shape();
MS_LOG(INFO) << "Tensor image shape size: " << shape.size();
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
EXPECT_GT(shape.size(), 1); // Verify decode=true took effect
iter->GetNextRow(&row);
}
@ -189,8 +187,8 @@ TEST_F(MindDataTestPipeline, TestAlbumNumSamplers) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape();
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row);
}

12
tests/ut/cpp/dataset/c_api_dataset_cifar_test.cc
Unescape View File

@ -48,8 +48,8 @@ TEST_F(MindDataTestPipeline, TestCifar10Dataset) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape();
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row);
}
@ -103,8 +103,8 @@ TEST_F(MindDataTestPipeline, TestCifar10DatasetWithPipeline) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape();
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row);
}
@ -187,8 +187,8 @@ TEST_F(MindDataTestPipeline, TestCifar100Dataset) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape();
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row);
}

43
tests/ut/cpp/dataset/c_api_dataset_clue_test.cc
Unescape View File

@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -53,12 +53,12 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetAFQMC) {
uint64_t i = 0;
while (row.size() != 0) {
// auto text = row["sentence1"];
auto text = row["sentence1"];
// std::string_view sv;
// text->GetItemAt(&sv, {0});
// std::string ss(sv);
// EXPECT_STREQ(ss.c_str(), expected_result[i].c_str());
// MS_LOG(INFO) << "Tensor text shape: " << text->shape();
MS_LOG(INFO) << "Tensor text shape: " << text.Shape();
iter->GetNextRow(&row);
i++;
}
@ -134,8 +134,8 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetBasic) {
EXPECT_NE(row.find("sentence1"), row.end());
uint64_t i = 0;
while (row.size() != 0) {
// auto text = row["sentence1"];
// MS_LOG(INFO) << "Tensor text shape: " << text->shape();
auto text = row["sentence1"];
MS_LOG(INFO) << "Tensor text shape: " << text.Shape();
i++;
iter->GetNextRow(&row);
}
@ -190,8 +190,8 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetBasicWithPipeline) {
EXPECT_NE(row.find("sentence1"), row.end());
uint64_t i = 0;
while (row.size() != 0) {
// auto text = row["sentence1"];
// MS_LOG(INFO) << "Tensor text shape: " << text->shape();
auto text = row["sentence1"];
MS_LOG(INFO) << "Tensor text shape: " << text.Shape();
i++;
iter->GetNextRow(&row);
}
@ -242,12 +242,12 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetCMNLI) {
uint64_t i = 0;
while (row.size() != 0) {
// auto text = row["sentence1"];
auto text = row["sentence1"];
// std::string_view sv;
// text->GetItemAt(&sv, {0});
// std::string ss(sv);
// EXPECT_STREQ(ss.c_str(), expected_result[i].c_str());
// MS_LOG(INFO) << "Tensor text shape: " << text->shape();
MS_LOG(INFO) << "Tensor text shape: " << text.Shape();
iter->GetNextRow(&row);
i++;
}
@ -283,12 +283,12 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetCSL) {
uint64_t i = 0;
while (row.size() != 0) {
// auto text = row["abst"];
auto text = row["abst"];
// std::string_view sv;
// text->GetItemAt(&sv, {0});
// std::string ss(sv);
// EXPECT_STREQ(ss.c_str(), expected_result[i].c_str());
// MS_LOG(INFO) << "Tensor text shape: " << text->shape();
MS_LOG(INFO) << "Tensor text shape: " << text.Shape();
iter->GetNextRow(&row);
i++;
}
@ -322,8 +322,8 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetDistribution) {
EXPECT_NE(row.find("sentence1"), row.end());
uint64_t i = 0;
while (row.size() != 0) {
// auto text = row["sentence1"];
// MS_LOG(INFO) << "Tensor text shape: " << text->shape();
auto text = row["sentence1"];
MS_LOG(INFO) << "Tensor text shape: " << text.Shape();
i++;
iter->GetNextRow(&row);
}
@ -424,12 +424,12 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetIFLYTEK) {
uint64_t i = 0;
while (row.size() != 0) {
// auto text = row["sentence"];
auto text = row["sentence"];
// std::string_view sv;
// text->GetItemAt(&sv, {0});
// std::string ss(sv);
// EXPECT_STREQ(ss.c_str(), expected_result[i].c_str());
// MS_LOG(INFO) << "Tensor text shape: " << text->shape();
MS_LOG(INFO) << "Tensor text shape: " << text.Shape();
iter->GetNextRow(&row);
i++;
}
@ -602,13 +602,12 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetShuffleGlobal) {
// "蚂蚁借呗等额还款能否换成先息后本"};
uint64_t i = 0;
while (row.size() != 0) {
// auto text = row["sentence1"];
// MS_LOG(INFO) << "Tensor text shape: " << text->shape();
auto text = row["sentence1"];
// std::string_view sv;
// text->GetItemAt(&sv, {0});
// std::string ss(sv);
// EXPECT_STREQ(ss.c_str(), expected_result[i].c_str());
// MS_LOG(INFO) << "Tensor text shape: " << text->shape();
MS_LOG(INFO) << "Tensor text shape: " << text.Shape();
i++;
iter->GetNextRow(&row);
}
@ -648,12 +647,12 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetTNEWS) {
uint64_t i = 0;
while (row.size() != 0) {
// auto text = row["sentence"];
auto text = row["sentence"];
// std::string_view sv;
// text->GetItemAt(&sv, {0});
// std::string ss(sv);
// EXPECT_STREQ(ss.c_str(), expected_result[i].c_str());
// MS_LOG(INFO) << "Tensor text shape: " << text->shape();
MS_LOG(INFO) << "Tensor text shape: " << text.Shape();
iter->GetNextRow(&row);
i++;
}
@ -690,12 +689,12 @@ TEST_F(MindDataTestPipeline, TestCLUEDatasetWSC) {
uint64_t i = 0;
while (row.size() != 0) {
// auto text = row["text"];
auto text = row["text"];
// std::string_view sv;
// text->GetItemAt(&sv, {0});
// std::string ss(sv);
// EXPECT_STREQ(ss.c_str(), expected_result[i].c_str());
// MS_LOG(INFO) << "Tensor text shape: " << text->shape();
MS_LOG(INFO) << "Tensor text shape: " << text.Shape();
iter->GetNextRow(&row);
i++;
}

24
tests/ut/cpp/dataset/c_api_dataset_coco_test.cc
Unescape View File

@ -45,12 +45,12 @@ TEST_F(MindDataTestPipeline, TestCocoDefault) {
uint64_t i = 0;
while (row.size() != 0) {
// auto image = row["image"];
// auto bbox = row["bbox"];
// auto category_id = row["category_id"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape();
// MS_LOG(INFO) << "Tensor bbox shape: " << bbox->shape();
// MS_LOG(INFO) << "Tensor category_id shape: " << category_id->shape();
auto image = row["image"];
auto bbox = row["bbox"];
auto category_id = row["category_id"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
MS_LOG(INFO) << "Tensor bbox shape: " << bbox.Shape();
MS_LOG(INFO) << "Tensor category_id shape: " << category_id.Shape();
iter->GetNextRow(&row);
i++;
}
@ -102,12 +102,12 @@ TEST_F(MindDataTestPipeline, TestCocoDefaultWithPipeline) {
uint64_t i = 0;
while (row.size() != 0) {
// auto image = row["image"];
// auto bbox = row["bbox"];
// auto category_id = row["category_id"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape();
// MS_LOG(INFO) << "Tensor bbox shape: " << bbox->shape();
// MS_LOG(INFO) << "Tensor category_id shape: " << category_id->shape();
auto image = row["image"];
auto bbox = row["bbox"];
auto category_id = row["category_id"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
MS_LOG(INFO) << "Tensor bbox shape: " << bbox.Shape();
MS_LOG(INFO) << "Tensor category_id shape: " << category_id.Shape();
iter->GetNextRow(&row);
i++;
}

2
tests/ut/cpp/dataset/c_api_dataset_csv_test.cc
Unescape View File

@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

58
tests/ut/cpp/dataset/c_api_dataset_iterator_test.cc
Unescape View File

@ -1,4 +1,3 @@
/**
* Copyright 2020-2021 Huawei Technologies Co., Ltd
*
@ -18,8 +17,6 @@
#include "minddata/dataset/include/datasets.h"
using namespace mindspore::dataset;
using mindspore::dataset::Tensor;
using mindspore::dataset::TensorShape;
class MindDataTestPipeline : public UT::DatasetOpTesting {
protected:
@ -43,14 +40,15 @@ TEST_F(MindDataTestPipeline, TestIteratorEmptyColumn) {
// Iterate the dataset and get each row
std::vector<mindspore::MSTensor> row;
iter->GetNextRow(&row);
// TensorShape expect0({32, 32, 3});
// TensorShape expect1({});
std::vector<int64_t> expect_image = {32, 32, 3};
std::vector<int64_t> expect_label = {};
uint64_t i = 0;
while (row.size() != 0) {
// MS_LOG(INFO) << "row[0]:" << row[0]->shape() << ", row[1]:" << row[1]->shape();
// EXPECT_EQ(expect0, row[0]->shape());
// EXPECT_EQ(expect1, row[1]->shape());
MS_LOG(INFO) << "row[0]:" << row[0].Shape() << ", row[1]:" << row[1].Shape();
EXPECT_EQ(expect_image, row[0].Shape());
EXPECT_EQ(expect_label, row[1].Shape());
iter->GetNextRow(&row);
i++;
}
@ -82,14 +80,14 @@ TEST_F(MindDataTestPipeline, TestIteratorOneColumn) {
// Iterate the dataset and get each row
std::vector<mindspore::MSTensor> row;
iter->GetNextRow(&row);
// TensorShape expect({2, 28, 28, 1});
std::vector<int64_t> expect_image = {2, 28, 28, 1};
uint64_t i = 0;
while (row.size() != 0) {
// for (auto &v : row) {
// MS_LOG(INFO) << "image shape:" << v->shape();
// EXPECT_EQ(expect, v->shape());
// }
for (auto &v : row) {
MS_LOG(INFO) << "image shape:" << v.Shape();
EXPECT_EQ(expect_image, v.Shape());
}
iter->GetNextRow(&row);
i++;
}
@ -120,16 +118,15 @@ TEST_F(MindDataTestPipeline, TestIteratorReOrder) {
// Iterate the dataset and get each row
std::vector<mindspore::MSTensor> row;
iter->GetNextRow(&row);
// TensorShape expect0({32, 32, 3});
// TensorShape expect1({});
std::vector<int64_t> expect_image = {32, 32, 3};
std::vector<int64_t> expect_label = {};
// Check if we will catch "label" before "image" in row
// std::vector<std::string> expect = {"label", "image"};
// Check "label" before "image" in row
uint64_t i = 0;
while (row.size() != 0) {
// MS_LOG(INFO) << "row[0]:" << row[0]->shape() << ", row[1]:" << row[1]->shape();
// EXPECT_EQ(expect1, row[0]->shape());
// EXPECT_EQ(expect0, row[1]->shape());
MS_LOG(INFO) << "row[0]:" << row[0].Shape() << ", row[1]:" << row[1].Shape();
EXPECT_EQ(expect_label, row[0].Shape());
EXPECT_EQ(expect_image, row[1].Shape());
iter->GetNextRow(&row);
i++;
}
@ -161,20 +158,19 @@ TEST_F(MindDataTestPipeline, TestIteratorTwoColumns) {
// Iterate the dataset and get each row
std::vector<mindspore::MSTensor> row;
iter->GetNextRow(&row);
// std::vector<TensorShape> expect = {TensorShape({173673}), TensorShape({1, 4}), TensorShape({173673}),
// TensorShape({1, 4}), TensorShape({147025}), TensorShape({1, 4}),
// TensorShape({211653}), TensorShape({1, 4})};
std::vector<std::vector<int64_t>> expect = {{173673}, {1, 4}, {173673}, {1, 4},
{147025}, {1, 4}, {211653}, {1, 4}};
uint64_t i = 0;
// uint64_t j = 0;
uint64_t j = 0;
while (row.size() != 0) {
// MS_LOG(INFO) << "row[0]:" << row[0]->shape() << ", row[1]:" << row[1]->shape();
// EXPECT_EQ(2, row.size());
// EXPECT_EQ(expect[j++], row[0]->shape());
// EXPECT_EQ(expect[j++], row[1]->shape());
MS_LOG(INFO) << "row[0]:" << row[0].Shape() << ", row[1]:" << row[1].Shape();
EXPECT_EQ(2, row.size());
EXPECT_EQ(expect[j++], row[0].Shape());
EXPECT_EQ(expect[j++], row[1].Shape());
iter->GetNextRow(&row);
i++;
// j = (j == expect.size()) ? 0 : j;
j = (j == expect.size()) ? 0 : j;
}
EXPECT_EQ(i, 8);
@ -222,8 +218,8 @@ TEST_F(MindDataTestPipeline, TestIteratorNumEpoch) {
EXPECT_EQ(inner_row_cnt, random_data_num_row);
}
EXPECT_EQ(total_row_cnt, random_data_num_row * num_epochs);
// this will go beyond the random_data_num_row*num_epoch limit, hence error code is expected
EXPECT_FALSE(iter->GetNextRow(&row));
// This will go beyond the random_data_num_row*num_epoch limit, hence error code is expected
EXPECT_ERROR(iter->GetNextRow(&row));
// Manually terminate the pipeline
iter->Stop();
}

30
tests/ut/cpp/dataset/c_api_dataset_manifest_test.cc
Unescape View File

@ -43,8 +43,8 @@ TEST_F(MindDataTestPipeline, TestManifestBasic) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape();
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row);
}
@ -95,8 +95,8 @@ TEST_F(MindDataTestPipeline, TestManifestBasicWithPipeline) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape();
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row);
}
@ -162,11 +162,11 @@ TEST_F(MindDataTestPipeline, TestManifestDecode) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto image = row["image"];
// auto shape = image->shape();
// MS_LOG(INFO) << "Tensor image shape size: " << shape.Size();
// MS_LOG(INFO) << "Tensor image shape: " << image->shape();
// EXPECT_GT(shape.Size(), 1); // Verify decode=true took effect
auto image = row["image"];
auto shape = image.Shape();
MS_LOG(INFO) << "Tensor image shape size: " << shape.size();
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
EXPECT_GT(shape.size(), 1); // Verify decode=true took effect
iter->GetNextRow(&row);
}
@ -196,8 +196,8 @@ TEST_F(MindDataTestPipeline, TestManifestEval) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape();
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row);
}
@ -241,8 +241,8 @@ TEST_F(MindDataTestPipeline, TestManifestClassIndex) {
// int32_t label_idx = 0;
while (row.size() != 0) {
i++;
// auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape();
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
// row["label"]->GetItemAt<int32_t>(&label_idx, {});
// MS_LOG(INFO) << "Tensor label value: " << label_idx;
// auto label_it = std::find(expected_label.begin(), expected_label.end(), label_idx);
@ -276,8 +276,8 @@ TEST_F(MindDataTestPipeline, TestManifestNumSamplers) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape();
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row);
}

16
tests/ut/cpp/dataset/c_api_dataset_minddata_test.cc
Unescape View File

@ -246,7 +246,7 @@ TEST_F(MindDataTestPipeline, TestMindDataSuccess6) {
EXPECT_NE(ds5, nullptr);
std::vector<std::shared_ptr<Dataset>> ds = {ds1, ds2, ds3, ds4, ds5, ds6};
// std::vector<int32_t> expected_samples = {5, 5, 2, 3, 3, 2};
std::vector<int32_t> expected_samples = {5, 5, 2, 3, 3, 2};
for (int32_t i = 0; i < ds.size(); i++) {
// Create an iterator over the result of the above dataset
@ -258,13 +258,13 @@ TEST_F(MindDataTestPipeline, TestMindDataSuccess6) {
std::unordered_map<std::string, mindspore::MSTensor> row;
iter->GetNextRow(&row);
// uint64_t j = 0;
// while (row.size() != 0) {
// j++;
// MS_LOG(INFO) << "Tensor label: " << *row["label"];
// iter->GetNextRow(&row);
// }
// EXPECT_EQ(j, expected_samples[i]);
uint64_t j = 0;
while (row.size() != 0) {
j++;
// MS_LOG(INFO) << "Tensor label: " << *row["label"];
iter->GetNextRow(&row);
}
EXPECT_EQ(j, expected_samples[i]);
// Manually terminate the pipeline
iter->Stop();

78
tests/ut/cpp/dataset/c_api_dataset_ops_test.cc
Unescape View File

@ -101,8 +101,8 @@ TEST_F(MindDataTestPipeline, TestBatchAndRepeat) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape();
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row);
}
@ -137,8 +137,8 @@ TEST_F(MindDataTestPipeline, TestBucketBatchByLengthSuccess1) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape();
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row);
}
// 2 batches of size 5
@ -174,8 +174,8 @@ TEST_F(MindDataTestPipeline, TestBucketBatchByLengthSuccess2) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape();
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row);
}
// With 2 boundaries, 3 buckets are created
@ -486,8 +486,8 @@ TEST_F(MindDataTestPipeline, TestConcatSuccess) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape();
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row);
}
@ -566,8 +566,8 @@ TEST_F(MindDataTestPipeline, TestConcatSuccess2) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape();
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row);
}
@ -780,8 +780,8 @@ TEST_F(MindDataTestPipeline, TestImageFolderBatchAndRepeat) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape();
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row);
}
@ -917,8 +917,8 @@ TEST_F(MindDataTestPipeline, TestProjectMap) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape();
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row);
}
@ -1055,9 +1055,9 @@ TEST_F(MindDataTestPipeline, TestProjectMapAutoInjection) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape();
// EXPECT_EQ(image->shape()[0], 30);
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
// EXPECT_EQ(image.Shape()[0], 30);
iter->GetNextRow(&row);
}
@ -1177,8 +1177,8 @@ TEST_F(MindDataTestPipeline, TestRenameSuccess) {
while (row.size() != 0) {
i++;
// auto image = row["col1"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape();
auto image = row["col1"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row);
}
@ -1221,8 +1221,8 @@ TEST_F(MindDataTestPipeline, TestRepeatDefault) {
break;
}
i++;
// auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape();
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row);
}
@ -1260,8 +1260,8 @@ TEST_F(MindDataTestPipeline, TestRepeatOne) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape();
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row);
}
@ -1344,8 +1344,8 @@ TEST_F(MindDataTestPipeline, TestShuffleDataset) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape();
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row);
}
@ -1380,8 +1380,8 @@ TEST_F(MindDataTestPipeline, TestSkipDataset) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape();
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row);
}
MS_LOG(INFO) << "Number of rows: " << i;
@ -1425,8 +1425,8 @@ TEST_F(MindDataTestPipeline, TestSkipTakeRepeat) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape();
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row);
}
MS_LOG(INFO) << "Number of rows: " << i;
@ -1497,8 +1497,8 @@ TEST_F(MindDataTestPipeline, TestTakeDatasetDefault) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape();
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row);
}
MS_LOG(INFO) << "Number of rows: " << i;
@ -1578,8 +1578,8 @@ TEST_F(MindDataTestPipeline, TestTakeDatasetNormal) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape();
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row);
}
MS_LOG(INFO) << "Number of rows: " << i;
@ -1632,8 +1632,8 @@ TEST_F(MindDataTestPipeline, TestTensorOpsAndMap) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape();
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row);
}
@ -1751,8 +1751,8 @@ TEST_F(MindDataTestPipeline, TestZipSuccess) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape();
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row);
}
@ -1843,8 +1843,8 @@ TEST_F(MindDataTestPipeline, TestZipSuccess2) {
uint64_t i = 0;
while (row.size() != 0) {
i++;
// auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape();
auto image = row["image"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row);
}

213
tests/ut/cpp/dataset/c_api_dataset_randomdata_test.cc
View File

File diff suppressed because it is too large Load Diff

46
tests/ut/cpp/dataset/c_api_dataset_textfile_test.cc
Unescape View File

@ -1,5 +1,5 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
* Copyright 2020-2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -58,8 +58,8 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetBasic) {
uint64_t i = 0;
while (row.size() != 0) {
// auto text = row["text"];
// MS_LOG(INFO) << "Tensor text shape: " << text->shape();
auto text = row["text"];
MS_LOG(INFO) << "Tensor text shape: " << text.Shape();
// std::string_view sv;
// text->GetItemAt(&sv, {0});
// std::string ss(sv);
@ -128,8 +128,8 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetBasicWithPipeline) {
uint64_t i = 0;
while (row.size() != 0) {
// auto text = row["text"];
// MS_LOG(INFO) << "Tensor text shape: " << text->shape();
auto text = row["text"];
MS_LOG(INFO) << "Tensor text shape: " << text.Shape();
i++;
iter->GetNextRow(&row);
}
@ -315,8 +315,8 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleFalse1A) {
uint64_t i = 0;
while (row.size() != 0) {
// auto text = row["text"];
// MS_LOG(INFO) << "Tensor text shape: " << text->shape();
auto text = row["text"];
MS_LOG(INFO) << "Tensor text shape: " << text.Shape();
// std::string_view sv;
// text->GetItemAt(&sv, {0});
// std::string ss(sv);
@ -373,8 +373,8 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleFalse1B) {
uint64_t i = 0;
while (row.size() != 0) {
// auto text = row["text"];
// MS_LOG(INFO) << "Tensor text shape: " << text->shape();
auto text = row["text"];
MS_LOG(INFO) << "Tensor text shape: " << text.Shape();
// std::string_view sv;
// text->GetItemAt(&sv, {0});
// std::string ss(sv);
@ -430,8 +430,8 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleFalse4Shard) {
uint64_t i = 0;
while (row.size() != 0) {
// auto text = row["text"];
// MS_LOG(INFO) << "Tensor text shape: " << text->shape();
auto text = row["text"];
MS_LOG(INFO) << "Tensor text shape: " << text.Shape();
// std::string_view sv;
// text->GetItemAt(&sv, {0});
// std::string ss(sv);
@ -490,8 +490,8 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleFiles1A) {
uint64_t i = 0;
while (row.size() != 0) {
// auto text = row["text"];
// MS_LOG(INFO) << "Tensor text shape: " << text->shape();
auto text = row["text"];
MS_LOG(INFO) << "Tensor text shape: " << text.Shape();
// std::string_view sv;
// text->GetItemAt(&sv, {0});
// std::string ss(sv);
@ -550,8 +550,8 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleFiles1B) {
uint64_t i = 0;
while (row.size() != 0) {
// auto text = row["text"];
// MS_LOG(INFO) << "Tensor text shape: " << text->shape();
auto text = row["text"];
MS_LOG(INFO) << "Tensor text shape: " << text.Shape();
// std::string_view sv;
// text->GetItemAt(&sv, {0});
// std::string ss(sv);
@ -609,8 +609,8 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleFiles4) {
uint64_t i = 0;
while (row.size() != 0) {
// auto text = row["text"];
// MS_LOG(INFO) << "Tensor text shape: " << text->shape();
auto text = row["text"];
MS_LOG(INFO) << "Tensor text shape: " << text.Shape();
// std::string_view sv;
// text->GetItemAt(&sv, {0});
// std::string ss(sv);
@ -664,8 +664,8 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleGlobal1A) {
uint64_t i = 0;
while (row.size() != 0) {
// auto text = row["text"];
// MS_LOG(INFO) << "Tensor text shape: " << text->shape();
auto text = row["text"];
MS_LOG(INFO) << "Tensor text shape: " << text.Shape();
// std::string_view sv;
// text->GetItemAt(&sv, {0});
// std::string ss(sv);
@ -722,8 +722,8 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleGlobal1B) {
uint64_t i = 0;
while (row.size() != 0) {
// auto text = row["text"];
// MS_LOG(INFO) << "Tensor text shape: " << text->shape();
auto text = row["text"];
MS_LOG(INFO) << "Tensor text shape: " << text.Shape();
// std::string_view sv;
// text->GetItemAt(&sv, {0});
// std::string ss(sv);
@ -780,8 +780,8 @@ TEST_F(MindDataTestPipeline, TestTextFileDatasetShuffleGlobal4) {
uint64_t i = 0;
while (row.size() != 0) {
// auto text = row["text"];
// MS_LOG(INFO) << "Tensor text shape: " << text->shape();
auto text = row["text"];
MS_LOG(INFO) << "Tensor text shape: " << text.Shape();
// std::string_view sv;
// text->GetItemAt(&sv, {0});
// std::string ss(sv);

51
tests/ut/cpp/dataset/c_api_dataset_tfrecord_test.cc
Unescape View File

@ -70,9 +70,9 @@ TEST_F(MindDataTestPipeline, TestTFRecordDatasetBasic) {
uint64_t i = 0;
while (row.size() != 0) {
// auto image = row["image"];
auto image = row["image"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape();
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
iter->GetNextRow(&row);
i++;
}
@ -279,23 +279,30 @@ TEST_F(MindDataTestPipeline, TestTFRecordDatasetSchemaObj) {
EXPECT_NE(row.find("col_float"), row.end());
EXPECT_NE(row.find("col_2d"), row.end());
std::vector<int64_t> expect_num = {1};
std::vector<int64_t> expect_2d = {2, 2};
uint64_t i = 0;
while (row.size() != 0) {
// auto col_sint16 = row["col_sint16"];
// auto col_float = row["col_float"];
// auto col_2d = row["col_2d"];
auto col_sint16 = row["col_sint16"];
auto col_float = row["col_float"];
auto col_2d = row["col_2d"];
// Validate shape
ASSERT_EQ(col_sint16.Shape(), expect_num);
ASSERT_EQ(col_float.Shape(), expect_num);
ASSERT_EQ(col_2d.Shape(), expect_2d);
// EXPECT_EQ(col_sint16->shape(), TensorShape({1}));
// EXPECT_EQ(col_float->shape(), TensorShape({1}));
// EXPECT_EQ(col_2d->shape(), TensorShape({2, 2}));
// Validate Rank
ASSERT_EQ(col_sint16.Shape().size(), 1);
ASSERT_EQ(col_float.Shape().size(), 1);
ASSERT_EQ(col_2d.Shape().size(), 2);
// EXPECT_EQ(col_sint16->Rank(), 1);
// EXPECT_EQ(col_float->Rank(), 1);
// EXPECT_EQ(col_2d->Rank(), 2);
// Validate type
ASSERT_EQ(col_sint16.DataType(), mindspore::DataType::kNumberTypeInt16);
ASSERT_EQ(col_float.DataType(), mindspore::DataType::kNumberTypeFloat32);
ASSERT_EQ(col_2d.DataType(), mindspore::DataType::kNumberTypeInt64);
// EXPECT_EQ(col_sint16->type(), DataType::DE_INT16);
// EXPECT_EQ(col_float->type(), DataType::DE_FLOAT32);
// EXPECT_EQ(col_2d->type(), DataType::DE_INT64);
iter->GetNextRow(&row);
i++;
}
@ -331,11 +338,11 @@ TEST_F(MindDataTestPipeline, TestTFRecordDatasetNoSchema) {
uint64_t i = 0;
while (row.size() != 0) {
// auto image = row["image"];
// auto label = row["label"];
auto image = row["image"];
auto label = row["label"];
// MS_LOG(INFO) << "Shape of column [image]:" << image->shape();
// MS_LOG(INFO) << "Shape of column [label]:" << label->shape();
MS_LOG(INFO) << "Shape of column [image]:" << image.Shape();
MS_LOG(INFO) << "Shape of column [label]:" << label.Shape();
iter->GetNextRow(&row);
i++;
}
@ -486,16 +493,16 @@ TEST_F(MindDataTestPipeline, TestIncorrectTFSchemaObject) {
EXPECT_NE(ds, nullptr);
auto itr = ds->CreateIterator();
EXPECT_NE(itr, nullptr);
// TensorMap mp;
// this will fail due to the incorrect schema used
// EXPECT_FALSE(itr->GetNextRow(&mp));
MSTensorMap mp;
// This will fail due to the incorrect schema used
EXPECT_ERROR(itr->GetNextRow(&mp));
}
TEST_F(MindDataTestPipeline, TestIncorrectTFrecordFile) {
std::string path = datasets_root_path_ + "/test_tf_file_3_images2/datasetSchema.json";
std::shared_ptr<Dataset> ds = TFRecord({path});
EXPECT_NE(ds, nullptr);
// the tf record file is incorrect, hence validate param will fail
// The tf record file is incorrect, hence validate param will fail
auto itr = ds->CreateIterator();
EXPECT_EQ(itr, nullptr);
}

24
tests/ut/cpp/dataset/c_api_dataset_voc_test.cc
Unescape View File

@ -55,10 +55,10 @@ TEST_F(MindDataTestPipeline, TestVOCClassIndex) {
// uint32_t expect[] = {9, 9, 9, 1, 1, 0};
uint64_t i = 0;
while (row.size() != 0) {
// auto image = row["image"];
// auto label = row["label"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape();
// MS_LOG(INFO) << "Tensor label shape: " << label->shape();
auto image = row["image"];
auto label = row["label"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
MS_LOG(INFO) << "Tensor label shape: " << label.Shape();
// expect_label->SetItemAt({0, 0}, expect[i]);
// EXPECT_EQ(*label, *expect_label);
@ -137,10 +137,10 @@ TEST_F(MindDataTestPipeline, TestVOCDetection) {
// uint32_t expect_num[] = {5, 5, 4, 3};
uint64_t i = 0;
while (row.size() != 0) {
// auto image = row["image"];
// auto label = row["label"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape();
// MS_LOG(INFO) << "Tensor label shape: " << label->shape();
auto image = row["image"];
auto label = row["label"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
MS_LOG(INFO) << "Tensor label shape: " << label.Shape();
// std::shared_ptr<Tensor> expect_image;
// Tensor::CreateFromFile(folder_path + "/JPEGImages/" + expect_file[i] + ".jpg", &expect_image);
@ -210,10 +210,10 @@ TEST_F(MindDataTestPipeline, TestVOCSegmentation) {
// std::string expect_file[] = {"32", "33", "39", "32", "33", "39"};
uint64_t i = 0;
while (row.size() != 0) {
// auto image = row["image"];
// auto target = row["target"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape();
// MS_LOG(INFO) << "Tensor target shape: " << target->shape();
auto image = row["image"];
auto target = row["target"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
MS_LOG(INFO) << "Tensor target shape: " << target.Shape();
// std::shared_ptr<Tensor> expect_image;
// Tensor::CreateFromFile(folder_path + "/JPEGImages/" + expect_file[i] + ".jpg", &expect_image);

8
tests/ut/cpp/dataset/c_api_datasets_test.cc
Unescape View File

@ -93,10 +93,10 @@ TEST_F(MindDataTestPipeline, TestCelebADefault) {
// Check if CelebAOp read correct images/attr
uint64_t i = 0;
while (row.size() != 0) {
// auto image = row["image"];
// auto attr = row["attr"];
// MS_LOG(INFO) << "Tensor image shape: " << image->shape();
// MS_LOG(INFO) << "Tensor attr shape: " << attr->shape();
auto image = row["image"];
auto attr = row["attr"];
MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
MS_LOG(INFO) << "Tensor attr shape: " << attr.Shape();
iter->GetNextRow(&row);
i++;

2
tests/ut/cpp/dataset/ir_vision_test.cc
Unescape View File

@ -329,7 +329,7 @@ TEST_F(MindDataTestIRVision, TestSoftDvppDecodeResizeJpegFail) {
Status rc;
// CSoftDvppDecodeResizeJpeg: size must be a vector of one or two values
// SoftDvppDecodeResizeJpeg: size must be a vector of one or two values
std::shared_ptr<TensorOperation> soft_dvpp_decode_resize_jpeg_op1(new vision::SoftDvppDecodeResizeJpegOperation({}));
rc = soft_dvpp_decode_resize_jpeg_op1->ValidateParams();
EXPECT_ERROR(rc);

Write Preview
Loading…
Cancel
Save