m53297601
/
mindspore
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

More understandable err msg when num rows is zero

Browse Source
pull/793/head
yanghaitao 5 years ago
parent 1f2ca74cd1
commit eb60ef00d1
8 changed files with 58 additions and 21 deletions
Show Stats Download Patch File Download Diff File

18
mindspore/ccsrc/dataset/engine/datasetops/source/celeba_op.cc
Unescape View File

@ -33,8 +33,8 @@ CelebAOp::Builder::Builder() : builder_decode_(false), builder_sampler_(nullptr)
} }
Status CelebAOp::Builder::Build(std::shared_ptr<CelebAOp> *op) { Status CelebAOp::Builder::Build(std::shared_ptr<CelebAOp> *op) {
MS_LOG(INFO) << "Celeba dataset directory is " << builder_dir_.c_str() << "."; MS_LOG(DEBUG) << "Celeba dataset directory is " << builder_dir_.c_str() << ".";
MS_LOG(INFO) << "Celeba dataset type is " << builder_dataset_type_.c_str() << "."; MS_LOG(DEBUG) << "Celeba dataset type is " << builder_dataset_type_.c_str() << ".";
RETURN_IF_NOT_OK(SanityCheck()); RETURN_IF_NOT_OK(SanityCheck());
if (builder_sampler_ == nullptr) { if (builder_sampler_ == nullptr) {
builder_sampler_ = std::make_shared<SequentialSampler>(); builder_sampler_ = std::make_shared<SequentialSampler>();
@ -240,9 +240,11 @@ Status CelebAOp::ParseImageAttrInfo() {
num_rows_exact_ = image_labels_vec_.size(); num_rows_exact_ = image_labels_vec_.size();
num_samples_ = (num_samples_ == 0 || num_samples_ > num_rows_exact_) ? num_rows_exact_ : num_samples_; num_samples_ = (num_samples_ == 0 || num_samples_ > num_rows_exact_) ? num_rows_exact_ : num_samples_;
if (num_rows_exact_ == 0) { if (num_rows_exact_ == 0) {
RETURN_STATUS_UNEXPECTED("Number of rows in celeba dataset is zero"); RETURN_STATUS_UNEXPECTED(
"There is no valid data matching the dataset API CelebADataset.Please check file path or dataset API "
"validation first.");
} }
MS_LOG(INFO) << "Celeba dataset rows number is " << num_rows_exact_ << "."; MS_LOG(DEBUG) << "Celeba dataset rows number is " << num_rows_exact_ << ".";
return Status::OK(); return Status::OK();
} }
@ -267,7 +269,9 @@ std::vector<std::string> CelebAOp::Split(const std::string &line) {
// Derived from RandomAccessOp // Derived from RandomAccessOp
Status CelebAOp::GetNumSamples(int64_t *num) const { Status CelebAOp::GetNumSamples(int64_t *num) const {
if (num == nullptr || num_samples_ == 0) { if (num == nullptr || num_samples_ == 0) {
RETURN_STATUS_UNEXPECTED("NumSample not set"); RETURN_STATUS_UNEXPECTED(
"There is no valid data matching the dataset API CelebADataset.Please check file path or dataset API "
"validation first.");
} }
(*num) = num_samples_; (*num) = num_samples_;
return Status::OK(); return Status::OK();
@ -275,7 +279,9 @@ Status CelebAOp::GetNumSamples(int64_t *num) const {
Status CelebAOp::GetNumRowsInDataset(int64_t *num) const { Status CelebAOp::GetNumRowsInDataset(int64_t *num) const {
if (num == nullptr || num_rows_exact_ == 0) { if (num == nullptr || num_rows_exact_ == 0) {
return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, "NumRow not set"); RETURN_STATUS_UNEXPECTED(
"There is no valid data matching the dataset API CelebADataset.Please check file path or dataset API "
"validation first.");
} }
*num = num_rows_exact_; *num = num_rows_exact_;

15
mindspore/ccsrc/dataset/engine/datasetops/source/cifar_op.cc
Unescape View File

@ -247,7 +247,10 @@ Status CifarOp::InitSampler() {
// Derived from RandomAccessOp // Derived from RandomAccessOp
Status CifarOp::GetNumSamples(int64_t *num) const { Status CifarOp::GetNumSamples(int64_t *num) const {
if (num == nullptr || num_rows_ == 0) { if (num == nullptr || num_rows_ == 0) {
RETURN_STATUS_UNEXPECTED("NumRow not set"); std::string api = cifar_type_ == kCifar10 ? "Cifar10Dataset" : "Cifar100Dataset";
std::string err_msg = "There is no valid data matching the dataset API " + api +
".Please check file path or dataset API validation first.";
RETURN_STATUS_UNEXPECTED(err_msg);
} }
(*num) = num_samples_; (*num) = num_samples_;
return Status::OK(); return Status::OK();
@ -256,7 +259,10 @@ Status CifarOp::GetNumSamples(int64_t *num) const {
// Derived from RandomAccessOp // Derived from RandomAccessOp
Status CifarOp::GetNumRowsInDataset(int64_t *num) const { Status CifarOp::GetNumRowsInDataset(int64_t *num) const {
if (num == nullptr || num_rows_ == 0) { if (num == nullptr || num_rows_ == 0) {
RETURN_STATUS_UNEXPECTED("NumRow not set"); std::string api = cifar_type_ == kCifar10 ? "Cifar10Dataset" : "Cifar100Dataset";
std::string err_msg = "There is no valid data matching the dataset API " + api +
".Please check file path or dataset API validation first.";
RETURN_STATUS_UNEXPECTED(err_msg);
} }
(*num) = num_rows_; (*num) = num_rows_;
return Status::OK(); return Status::OK();
@ -389,7 +395,10 @@ Status CifarOp::ParseCifarData() {
num_rows_ = cifar_image_label_pairs_.size(); num_rows_ = cifar_image_label_pairs_.size();
num_samples_ = (num_samples_ == 0 || num_samples_ > num_rows_) ? num_rows_ : num_samples_; num_samples_ = (num_samples_ == 0 || num_samples_ > num_rows_) ? num_rows_ : num_samples_;
if (num_rows_ == 0) { if (num_rows_ == 0) {
RETURN_STATUS_UNEXPECTED("Init Cifar failed, not a single row read from dataset!"); std::string api = cifar_type_ == kCifar10 ? "Cifar10Dataset" : "Cifar100Dataset";
std::string err_msg = "There is no valid data matching the dataset API " + api +
".Please check file path or dataset API validation first.";
RETURN_STATUS_UNEXPECTED(err_msg);
} }
cifar_raw_data_block_->Reset(); cifar_raw_data_block_->Reset();
return Status::OK(); return Status::OK();

8
mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.cc
Unescape View File

@ -265,7 +265,9 @@ Status ImageFolderOp::InitSampler() {
// Derived from RandomAccessOp // Derived from RandomAccessOp
Status ImageFolderOp::GetNumSamples(int64_t *num) const { Status ImageFolderOp::GetNumSamples(int64_t *num) const {
if (num == nullptr || num_samples_ == 0) { if (num == nullptr || num_samples_ == 0) {
RETURN_STATUS_UNEXPECTED("NumRow not set"); RETURN_STATUS_UNEXPECTED(
"There is no valid data matching the dataset API ImageFolderDatasetV2.Please check file path or dataset API "
"validation first.");
} }
(*num) = num_samples_; (*num) = num_samples_;
return Status::OK(); return Status::OK();
@ -274,7 +276,9 @@ Status ImageFolderOp::GetNumSamples(int64_t *num) const {
// Derived from RandomAccessOp // Derived from RandomAccessOp
Status ImageFolderOp::GetNumRowsInDataset(int64_t *num) const { Status ImageFolderOp::GetNumRowsInDataset(int64_t *num) const {
if (num == nullptr || num_rows_ == 0) { if (num == nullptr || num_rows_ == 0) {
RETURN_STATUS_UNEXPECTED("NumRow not set"); RETURN_STATUS_UNEXPECTED(
"There is no valid data matching the dataset API ImageFolderDatasetV2.Please check file path or dataset API "
"validation first.");
} }
(*num) = num_rows_; (*num) = num_rows_;
return Status::OK(); return Status::OK();

14
mindspore/ccsrc/dataset/engine/datasetops/source/manifest_op.cc
Unescape View File

@ -261,7 +261,9 @@ Status ManifestOp::InitSampler() {
// Derived from RandomAccessOp // Derived from RandomAccessOp
Status ManifestOp::GetNumSamples(int64_t *num) const { Status ManifestOp::GetNumSamples(int64_t *num) const {
if (num == nullptr || num_rows_ == 0) { if (num == nullptr || num_rows_ == 0) {
RETURN_STATUS_UNEXPECTED("NumRow not set"); RETURN_STATUS_UNEXPECTED(
"There is no valid data matching the dataset API ManifestDataset.Please check file path or dataset API "
"validation first.");
} }
(*num) = num_samples_; (*num) = num_samples_;
return Status::OK(); return Status::OK();
@ -270,7 +272,9 @@ Status ManifestOp::GetNumSamples(int64_t *num) const {
// Derived from RandomAccessOp // Derived from RandomAccessOp
Status ManifestOp::GetNumRowsInDataset(int64_t *num) const { Status ManifestOp::GetNumRowsInDataset(int64_t *num) const {
if (num == nullptr || num_rows_ == 0) { if (num == nullptr || num_rows_ == 0) {
RETURN_STATUS_UNEXPECTED("NumRow not set"); RETURN_STATUS_UNEXPECTED(
"There is no valid data matching the dataset API ManifestDataset.Please check file path or dataset API "
"validation first.");
} }
(*num) = num_rows_; (*num) = num_rows_;
return Status::OK(); return Status::OK();
@ -279,7 +283,7 @@ Status ManifestOp::GetNumRowsInDataset(int64_t *num) const {
// Derived from RandomAccessOp // Derived from RandomAccessOp
Status ManifestOp::GetClassIds(std::map<int32_t, std::vector<int64_t>> *cls_ids) const { Status ManifestOp::GetClassIds(std::map<int32_t, std::vector<int64_t>> *cls_ids) const {
if (cls_ids == nullptr || !cls_ids->empty() || image_labelname_.empty()) { if (cls_ids == nullptr || !cls_ids->empty() || image_labelname_.empty()) {
RETURN_STATUS_UNEXPECTED("Number rows is 0"); RETURN_STATUS_UNEXPECTED("Class indexing is invalid.");
} }
for (size_t i = 0; i < image_labelname_.size(); i++) { for (size_t i = 0; i < image_labelname_.size(); i++) {
@ -395,7 +399,9 @@ Status ManifestOp::CountDatasetInfo() {
num_rows_ = static_cast<int64_t>(image_labelname_.size()); num_rows_ = static_cast<int64_t>(image_labelname_.size());
num_samples_ = (num_samples_ == 0 || num_samples_ > num_rows_) ? num_rows_ : num_samples_; num_samples_ = (num_samples_ == 0 || num_samples_ > num_rows_) ? num_rows_ : num_samples_;
if (num_rows_ == 0) { if (num_rows_ == 0) {
RETURN_STATUS_UNEXPECTED("Number of rows is 0"); RETURN_STATUS_UNEXPECTED(
"There is no valid data matching the dataset API ManifestDataset.Please check file path or dataset API "
"validation first.");
} }
return Status::OK(); return Status::OK();
} }

8
mindspore/ccsrc/dataset/engine/datasetops/source/mnist_op.cc
Unescape View File

@ -212,7 +212,9 @@ Status MnistOp::InitSampler() {
// Derived from RandomAccessOp // Derived from RandomAccessOp
Status MnistOp::GetNumSamples(int64_t *num) const { Status MnistOp::GetNumSamples(int64_t *num) const {
if (num == nullptr || num_rows_ == 0) { if (num == nullptr || num_rows_ == 0) {
RETURN_STATUS_UNEXPECTED("NumRow not set"); RETURN_STATUS_UNEXPECTED(
"There is no valid data matching the dataset API MnistDataset.Please check file path or dataset API "
"validation first.");
} }
(*num) = num_samples_; (*num) = num_samples_;
return Status::OK(); return Status::OK();
@ -436,7 +438,9 @@ Status MnistOp::CountTotalRows(const std::string &dir, int64_t numSamples, int64
// Derived from RandomAccessOp // Derived from RandomAccessOp
Status MnistOp::GetNumRowsInDataset(int64_t *num) const { Status MnistOp::GetNumRowsInDataset(int64_t *num) const {
if (num == nullptr || num_rows_ == 0) { if (num == nullptr || num_rows_ == 0) {
RETURN_STATUS_UNEXPECTED("NumRow not set"); RETURN_STATUS_UNEXPECTED(
"There is no valid data matching the dataset API MnistDataset.Please check file path or dataset API "
"validation first.");
} }
(*num) = num_rows_; (*num) = num_rows_;
return Status::OK(); return Status::OK();

4
mindspore/ccsrc/dataset/engine/datasetops/source/text_file_op.cc
Unescape View File

@ -438,7 +438,9 @@ Status TextFileOp::CalculateNumRowsPerShard() {
all_num_rows_ += count; all_num_rows_ += count;
} }
if (all_num_rows_ == 0) { if (all_num_rows_ == 0) {
RETURN_STATUS_UNEXPECTED("Number of rows can not be zero"); RETURN_STATUS_UNEXPECTED(
"There is no valid data matching the dataset API TextFileDataset.Please check file path or dataset API "
"validation first.");
} }
num_rows_per_shard_ = static_cast<int64_t>(std::ceil(all_num_rows_ * 1.0 / num_devices_)); num_rows_per_shard_ = static_cast<int64_t>(std::ceil(all_num_rows_ * 1.0 / num_devices_));

4
mindspore/ccsrc/dataset/engine/datasetops/source/tf_reader_op.cc
Unescape View File

@ -198,7 +198,9 @@ Status TFReaderOp::CalculateNumRowsPerShard() {
} }
num_rows_per_shard_ = static_cast<int64_t>(std::ceil(num_rows_ * 1.0 / num_devices_)); num_rows_per_shard_ = static_cast<int64_t>(std::ceil(num_rows_ * 1.0 / num_devices_));
if (num_rows_per_shard_ == 0) { if (num_rows_per_shard_ == 0) {
RETURN_STATUS_UNEXPECTED("Number of rows can not be zero"); RETURN_STATUS_UNEXPECTED(
"There is no valid data matching the dataset API TFRecordDataset.Please check file path or dataset API "
"validation first.");
} }
return Status::OK(); return Status::OK();
} }

8
mindspore/ccsrc/dataset/engine/datasetops/source/voc_op.cc
Unescape View File

@ -147,7 +147,9 @@ Status VOCOp::Reset() {
Status VOCOp::GetNumSamples(int64_t *num) const { Status VOCOp::GetNumSamples(int64_t *num) const {
if (num == nullptr || num_rows_ == 0) { if (num == nullptr || num_rows_ == 0) {
RETURN_STATUS_UNEXPECTED("NumRow not set"); RETURN_STATUS_UNEXPECTED(
"There is no valid data matching the dataset API VOCDataset.Please check file path or dataset API "
"validation first.");
} }
(*num) = num_samples_; (*num) = num_samples_;
return Status::OK(); return Status::OK();
@ -261,7 +263,9 @@ Status VOCOp::ReadImageToTensor(const std::string &path, const ColDescriptor &co
// Derived from RandomAccessOp // Derived from RandomAccessOp
Status VOCOp::GetNumRowsInDataset(int64_t *num) const { Status VOCOp::GetNumRowsInDataset(int64_t *num) const {
if (num == nullptr || num_rows_ == 0) { if (num == nullptr || num_rows_ == 0) {
RETURN_STATUS_UNEXPECTED("NumRow not set"); RETURN_STATUS_UNEXPECTED(
"There is no valid data matching the dataset API VOCDataset.Please check file path or dataset API "
"validation first.");
} }
(*num) = num_rows_; (*num) = num_rows_;
return Status::OK(); return Status::OK();

Write Preview
Loading…
Cancel
Save