m53297601
/
mindspore
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

!4402 [Dataset] Fix c_api bugs

Browse Source
pull/4402/head
luoyang 5 years ago
parent d3ef9f77c5
commit 373adaa553
5 changed files with 63 additions and 20 deletions
Show Stats Download Patch File Download Diff File

14
mindspore/ccsrc/minddata/dataset/api/datasets.cc
Unescape View File

@ -397,6 +397,10 @@ bool ValidateCommonDatasetParams(std::string dataset_dir) {
MS_LOG(ERROR) << "No dataset path is specified";
return false;
}
if (access(dataset_dir.c_str(), R_OK) == -1) {
MS_LOG(ERROR) << "No access to specified dataset path: " << dataset_dir;
return false;
}
return true;
}
@ -1176,8 +1180,8 @@ std::vector<std::shared_ptr<DatasetOp>> RepeatDataset::Build() {
}
bool RepeatDataset::ValidateParams() {
if (repeat_count_ != -1 && repeat_count_ <= 0) {
MS_LOG(ERROR) << "Repeat: Repeat count cannot be" << repeat_count_;
if (repeat_count_ <= 0 && repeat_count_ != -1) {
MS_LOG(ERROR) << "Repeat: repeat_count should be either -1 or positive integer, repeat_count_: " << repeat_count_;
return false;
}
@ -1223,7 +1227,7 @@ std::vector<std::shared_ptr<DatasetOp>> SkipDataset::Build() {
// Function to validate the parameters for SkipDataset
bool SkipDataset::ValidateParams() {
if (skip_count_ <= -1) {
MS_LOG(ERROR) << "Skip: Invalid input, skip_count: " << skip_count_;
MS_LOG(ERROR) << "Skip: skip_count should not be negative, skip_count: " << skip_count_;
return false;
}
@ -1244,8 +1248,8 @@ std::vector<std::shared_ptr<DatasetOp>> TakeDataset::Build() {
// Function to validate the parameters for TakeDataset
bool TakeDataset::ValidateParams() {
if (take_count_ < -1) {
MS_LOG(ERROR) << "Take: Invalid input, take_count: " << take_count_;
if (take_count_ < 0 && take_count_ != -1) {
MS_LOG(ERROR) << "Take: take_count should be either -1 or positive integer, take_count: " << take_count_;
return false;
}

4
mindspore/ccsrc/minddata/dataset/api/iterator.cc
Unescape View File

@ -25,7 +25,7 @@ namespace api {
void Iterator::GetNextRow(TensorMap *row) {
Status rc = iterator_->GetNextAsMap(row);
if (rc.IsError()) {
MS_LOG(ERROR) << "GetNextRow: Failed to get next row.";
MS_LOG(ERROR) << "GetNextRow: Failed to get next row. Error status: " << rc;
row->clear();
}
}
@ -35,7 +35,7 @@ void Iterator::GetNextRow(TensorVec *row) {
TensorRow tensor_row;
Status rc = iterator_->FetchNextTensorRow(&tensor_row);
if (rc.IsError()) {
MS_LOG(ERROR) << "GetNextRow: Failed to get next row.";
MS_LOG(ERROR) << "GetNextRow: Failed to get next row. Error status: " << rc;
row->clear();
}
// Generate a vector as return

1
mindspore/ccsrc/minddata/dataset/include/datasets.h
Unescape View File

@ -17,6 +17,7 @@
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASETS_H_
#define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_DATASETS_H_
#include <unistd.h>
#include <vector>
#include <memory>
#include <set>

42
tests/ut/cpp/dataset/c_api_dataset_ops_test.cc
Unescape View File

@ -437,22 +437,22 @@ TEST_F(MindDataTestPipeline, TestRepeatDefault) {
// Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr <Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
EXPECT_NE(ds,nullptr);
EXPECT_NE(ds, nullptr);
// Create a Repeat operation on ds
// Default value of repeat count is -1, expected to repeat infinitely
ds = ds->Repeat();
EXPECT_NE(ds,nullptr);
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
int32_t batch_size = 1;
ds = ds->Batch(batch_size);
EXPECT_NE(ds,nullptr);
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
// This will trigger the creation of the Execution Tree and launch it.
std::shared_ptr <Iterator> iter = ds->CreateIterator();
EXPECT_NE(iter,nullptr);
EXPECT_NE(iter, nullptr);
// Iterate the dataset and get each row
std::unordered_map <std::string, std::shared_ptr<Tensor>> row;
@ -460,14 +460,16 @@ TEST_F(MindDataTestPipeline, TestRepeatDefault) {
uint64_t i = 0;
while (row.size()!= 0) {
// manually stop
if(i==100){break;}
if (i == 100) {
break;
}
i++;
auto image = row["image"];
MS_LOG(INFO)<< "Tensor image shape: " << image->shape();
iter->GetNextRow(&row);
}
EXPECT_EQ(i,100);
EXPECT_EQ(i, 100);
// Manually terminate the pipeline
iter->Stop();
}
@ -478,22 +480,22 @@ TEST_F(MindDataTestPipeline, TestRepeatOne) {
// Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr <Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
EXPECT_NE(ds,nullptr);
EXPECT_NE(ds, nullptr);
// Create a Repeat operation on ds
int32_t repeat_num = 1;
ds = ds->Repeat(repeat_num);
EXPECT_NE(ds,nullptr);
EXPECT_NE(ds, nullptr);
// Create a Batch operation on ds
int32_t batch_size = 1;
ds = ds->Batch(batch_size);
EXPECT_NE(ds,nullptr);
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
// This will trigger the creation of the Execution Tree and launch it.
std::shared_ptr <Iterator> iter = ds->CreateIterator();
EXPECT_NE(iter,nullptr);
EXPECT_NE(iter, nullptr);
// Iterate the dataset and get each row
std::unordered_map <std::string, std::shared_ptr<Tensor>> row;
@ -506,13 +508,27 @@ TEST_F(MindDataTestPipeline, TestRepeatOne) {
iter->GetNextRow(&row);
}
EXPECT_EQ(i,10);
EXPECT_EQ(i, 10);
// Manually terminate the pipeline
iter->Stop();
}
TEST_F(MindDataTestPipeline, TestRepeatFail) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRepeatFail.";
TEST_F(MindDataTestPipeline, TestRepeatFail1) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRepeatFail1.";
// Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 10));
EXPECT_NE(ds, nullptr);
// Create a Repeat operation on ds
int32_t repeat_num = 0;
ds = ds->Repeat(repeat_num);
EXPECT_EQ(ds, nullptr);
}
TEST_F(MindDataTestPipeline, TestRepeatFail2) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRepeatFail2.";
// This case is expected to fail because the repeat count is invalid (<-1 && !=0).
// Create an ImageFolder Dataset

22
tests/ut/cpp/dataset/c_api_datasets_test.cc
Unescape View File

@ -133,3 +133,25 @@ TEST_F(MindDataTestPipeline, TestMnistFail1) {
std::shared_ptr<Dataset> ds = Mnist("", RandomSampler(false, 10));
EXPECT_EQ(ds, nullptr);
}
TEST_F(MindDataTestPipeline, TestImageFolderFail2) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestImageFolderFail2.";
// Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, RandomSampler(false, 2), {".JGP"});
EXPECT_NE(ds, nullptr);
// Create an iterator over the result of the above dataset
// This will trigger the creation of the Execution Tree and launch it.
std::shared_ptr<Iterator> iter = ds->CreateIterator();
EXPECT_NE(iter, nullptr);
// Iterate the dataset and get each row
std::unordered_map<std::string, std::shared_ptr<Tensor>> row;
iter->GetNextRow(&row);
EXPECT_EQ(row.size(), 0);
// Manually terminate the pipeline
iter->Stop();
}

Write Preview
Loading…
Cancel
Save