m53297601
/
mindspore
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

!7663 C++ API: ValidateParams support for TensorOps; plus minor dataset fixes

Browse Source 
Merge pull request !7663 from cathwong/ckw_cppapi_misc_cleanup_valparms_ir
pull/7663/MERGE
mindspore-ci-bot 4 years ago committed by Gitee
parent 79b5fee04b 051bc60edb
commit 7276198580
13 changed files with 357 additions and 263 deletions
Show Stats Download Patch File Download Diff File

16
mindspore/ccsrc/minddata/dataset/api/text.cc
Unescape View File

@ -48,17 +48,21 @@ LookupOperation::LookupOperation(const std::shared_ptr<Vocab> &vocab, const std:
const DataType &data_type)
: vocab_(vocab), unknown_token_(unknown_token), default_id_(Vocab::kNoTokenExists), data_type_(data_type) {}
bool LookupOperation::ValidateParams() {
Status LookupOperation::ValidateParams() {
if (vocab_ == nullptr) {
MS_LOG(ERROR) << "Lookup: vocab object type is incorrect or null.";
return false;
std::string err_msg = "Lookup: vocab object type is incorrect or null.";
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
default_id_ = vocab_->Lookup(unknown_token_);
if (default_id_ == Vocab::kNoTokenExists) {
MS_LOG(ERROR) << "Lookup: " << unknown_token_ << " doesn't exist in vocab.";
return false;
std::string err_msg = "Lookup: " + unknown_token_ + " doesn't exist in vocab.";
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
return true;
return Status::OK();
}
std::shared_ptr<TensorOp> LookupOperation::Build() {

19
mindspore/ccsrc/minddata/dataset/api/transforms.cc
Unescape View File

@ -62,13 +62,15 @@ std::shared_ptr<TypeCastOperation> TypeCast(std::string data_type) {
// OneHotOperation
OneHotOperation::OneHotOperation(int32_t num_classes) : num_classes_(num_classes) {}
bool OneHotOperation::ValidateParams() {
Status OneHotOperation::ValidateParams() {
if (num_classes_ < 0) {
MS_LOG(ERROR) << "OneHot: Number of classes cannot be negative. Number of classes: " << num_classes_;
return false;
std::string err_msg =
"OneHot: Number of classes cannot be negative. Number of classes: " + std::to_string(num_classes_);
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
return true;
return Status::OK();
}
std::shared_ptr<TensorOp> OneHotOperation::Build() { return std::make_shared<OneHotOp>(num_classes_); }
@ -76,17 +78,18 @@ std::shared_ptr<TensorOp> OneHotOperation::Build() { return std::make_shared<One
// TypeCastOperation
TypeCastOperation::TypeCastOperation(std::string data_type) : data_type_(data_type) {}
bool TypeCastOperation::ValidateParams() {
Status TypeCastOperation::ValidateParams() {
std::vector<std::string> predefine_type = {"bool", "int8", "uint8", "int16", "uint16", "int32", "uint32",
"int64", "uint64", "float16", "float32", "float64", "string"};
auto itr = std::find(predefine_type.begin(), predefine_type.end(), data_type_);
if (itr == predefine_type.end()) {
MS_LOG(ERROR) << "TypeCast: Only support type bool, int8, uint8, int16, uint16, int32, uint32, "
std::string err_msg = "TypeCast: Invalid data type: " + data_type_;
MS_LOG(ERROR) << "TypeCast: Only supports data type bool, int8, uint8, int16, uint16, int32, uint32, "
<< "int64, uint64, float16, float32, float64, string, but got " << data_type_;
return false;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
return true;
return Status::OK();
}
std::shared_ptr<TensorOp> TypeCastOperation::Build() { return std::make_shared<TypeCastOp>(data_type_); }

474
mindspore/ccsrc/minddata/dataset/api/vision.cc
View File

File diff suppressed because it is too large Load Diff

6
mindspore/ccsrc/minddata/dataset/engine/consumers/python_tree_consumer.h
Unescape View File

@ -13,8 +13,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_ENGINE_CONSUMER_PYTHON_TREE_CONSUMER_H_
#define MINDSPORE_CCSRC_MINDDATA_DATASET_ENGINE_CONSUMER_PYTHON_TREE_CONSUMER_H_
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_ENGINE_CONSUMERS_PYTHON_TREE_CONSUMER_H_
#define MINDSPORE_CCSRC_MINDDATA_DATASET_ENGINE_CONSUMERS_PYTHON_TREE_CONSUMER_H_
#include <memory>
#include <string>
@ -46,4 +46,4 @@ class PythonIterator : public IteratorConsumer {
};
} // namespace mindspore::dataset
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_ENGINE_CONSUMER_PYTHON_TREE_CONSUMER_H_
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_ENGINE_CONSUMERS_PYTHON_TREE_CONSUMER_H_

6
mindspore/ccsrc/minddata/dataset/engine/consumers/tree_consumer.h
Unescape View File

@ -13,8 +13,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_ENGINE_CONSUMER_TREE_CONSUMER_H_
#define MINDSPORE_CCSRC_MINDDATA_DATASET_ENGINE_CONSUMER_TREE_CONSUMER_H_
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_ENGINE_CONSUMERS_TREE_CONSUMER_H_
#define MINDSPORE_CCSRC_MINDDATA_DATASET_ENGINE_CONSUMERS_TREE_CONSUMER_H_
#include <memory>
#include <string>
@ -176,4 +176,4 @@ class TreeGetters : public TreeConsumer {
};
} // namespace mindspore::dataset
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_ENGINE_CONSUMER_TREE_CONSUMER_H_
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_ENGINE_CONSUMERS_TREE_CONSUMER_H_

4
mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/batch_node.cc
Unescape View File

@ -41,12 +41,12 @@ BatchNode::BatchNode(std::shared_ptr<Dataset> child, int32_t batch_size, bool dr
Status BatchNode::ValidateParams() {
if (batch_size_ <= 0) {
std::string err_msg = "Batch: batch_size should be positive integer, but got: " + std::to_string(batch_size_);
std::string err_msg = "BatchNode: batch_size should be positive integer, but got: " + std::to_string(batch_size_);
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
if (!cols_to_map_.empty()) {
std::string err_msg = "cols_to_map functionality is not implemented in C++; this should be left empty.";
std::string err_msg = "BatchNode: cols_to_map functionality is not implemented in C++; this should be left empty.";
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}

5
mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/bucket_batch_by_length_node.cc
Unescape View File

@ -62,8 +62,9 @@ std::vector<std::shared_ptr<DatasetOp>> BucketBatchByLengthNode::Build() {
Status BucketBatchByLengthNode::ValidateParams() {
if (element_length_function_ == nullptr && column_names_.size() != 1) {
std::string err_msg = "BucketBatchByLengthNode: element_length_function not specified, but not one column name: " +
std::to_string(column_names_.size());
std::string err_msg =
"BucketBatchByLengthNode: when element_length_function is not specified, size of column_name must be 1 but is: " +
std::to_string(column_names_.size());
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}

10
mindspore/ccsrc/minddata/dataset/engine/tree_adapter.cc
Unescape View File

@ -23,11 +23,11 @@ namespace mindspore {
namespace dataset {
Status TreeAdapter::BuildAndPrepare(std::shared_ptr<api::Dataset> root_ir, int32_t num_epoch) {
// Check whether this function has been called before. If so, return fail
// Check whether this function has been called before. If so, return failure
CHECK_FAIL_RETURN_UNEXPECTED(tree_ == nullptr, "ExecutionTree is already built.");
RETURN_UNEXPECTED_IF_NULL(root_ir);
// this will evolve in the long run
// This will evolve in the long run
tree_ = std::make_unique<ExecutionTree>();
std::shared_ptr<DatasetOp> root_op;
@ -37,7 +37,7 @@ Status TreeAdapter::BuildAndPrepare(std::shared_ptr<api::Dataset> root_ir, int32
// Prepare the tree
RETURN_IF_NOT_OK(tree_->Prepare(num_epoch));
// after the tree is prepared, the col_name_id_map can safely be obtained
// After the tree is prepared, the col_name_id_map can safely be obtained
column_name_map_ = tree_->root()->column_name_id_map();
return Status::OK();
@ -47,7 +47,7 @@ Status TreeAdapter::GetNext(TensorRow *row) {
RETURN_UNEXPECTED_IF_NULL(tree_);
RETURN_UNEXPECTED_IF_NULL(row);
row->clear(); // make sure row is empty
// cur_db_ being a nullptr means this is the first call to get_next, launch ExecutionTree
// When cur_db_ is a nullptr, it means this is the first call to get_next, launch ExecutionTree
if (cur_db_ == nullptr) {
RETURN_IF_NOT_OK(tree_->Launch());
RETURN_IF_NOT_OK(tree_->root()->GetNextBuffer(&cur_db_)); // first buf can't be eof or empty buf with none flag
@ -77,7 +77,7 @@ Status TreeAdapter::DFSBuildTree(std::shared_ptr<api::Dataset> ir, std::shared_p
RETURN_IF_NOT_OK(ops[i - 1]->AddChild(ops[i]));
}
// build the children of ir, once they return, add the return value to *op
// Build the children of ir, once they return, add the return value to *op
for (std::shared_ptr<api::Dataset> child_ir : ir->children) {
std::shared_ptr<DatasetOp> child_op;
RETURN_IF_NOT_OK(DFSBuildTree(child_ir, &child_op));

8
mindspore/ccsrc/minddata/dataset/engine/tree_adapter.h
Unescape View File

@ -37,7 +37,7 @@ class TreeAdapter {
~TreeAdapter() = default;
// This will construct a ExeTree from a Dataset root and Prepare() the ExeTree
// This will construct an ExeTree from a Dataset root and Prepare() the ExeTree
// This function is only meant to be called once and needs to be called before GetNext
// ExeTree will be launched when the first GetNext is called
Status BuildAndPrepare(std::shared_ptr<api::Dataset> root, int32_t num_epoch = -1);
@ -47,15 +47,15 @@ class TreeAdapter {
// 2. GetNext will return empty row when eoe/eof is obtained
Status GetNext(TensorRow *);
// this function will return the column_name_map once BuildAndPrepare() is called
// This function will return the column_name_map once BuildAndPrepare() is called
std::unordered_map<std::string, int32_t> GetColumnNameMap() const { return column_name_map_; }
// this function returns the TaskGroup associated with ExeTree, this is needed by DeviceQueueConsumer
// This function returns the TaskGroup associated with ExeTree. This is needed by DeviceQueueConsumer
// to be able to launch a thread. BuildAndPrepare needs to be called before this function
TaskGroup *AllTasks() const { return tree_ != nullptr ? tree_->AllTasks() : nullptr; }
private:
// this RECURSIVE function converts IR nodes into DatasetOp in ExecutionTree. ir could build a vector of ops. In
// This RECURSIVE function converts IR nodes into DatasetOp in ExecutionTree. IR could build a vector of ops. In
// such case, the first node is returned. Op is added as child when the current function returns.
Status DFSBuildTree(std::shared_ptr<api::Dataset> ir, std::shared_ptr<DatasetOp> *op);

1
mindspore/ccsrc/minddata/dataset/include/status.h
Unescape View File

@ -89,6 +89,7 @@ enum class StatusCode : char {
kTimeOut = 14,
kBuddySpaceFull = 15,
kNetWorkError = 16,
kNotImplementedYet = 17,
// Make this error code the last one. Add new error code above it.
kUnexpectedError = 127
};

5
mindspore/ccsrc/minddata/dataset/include/text.h
Unescape View File

@ -17,13 +17,14 @@
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_TEXT_H_
#define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_TEXT_H_
#include <vector>
#include <memory>
#include <string>
#include <vector>
#include "minddata/dataset/core/constants.h"
#include "minddata/dataset/include/transforms.h"
#include "minddata/dataset/text/vocab.h"
#include "minddata/dataset/util/status.h"
#include "mindspore/ccsrc/minddata/dataset/core/data_type.h"
namespace mindspore {
@ -56,7 +57,7 @@ class LookupOperation : public TensorOperation {
std::shared_ptr<TensorOp> Build() override;
bool ValidateParams() override;
Status ValidateParams() override;
private:
std::shared_ptr<Vocab> vocab_;

9
mindspore/ccsrc/minddata/dataset/include/transforms.h
Unescape View File

@ -17,10 +17,11 @@
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_TRANSFORMS_H_
#define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_TRANSFORMS_H_
#include <vector>
#include <memory>
#include <string>
#include <vector>
#include "minddata/dataset/core/constants.h"
#include "minddata/dataset/util/status.h"
namespace mindspore {
namespace dataset {
@ -41,7 +42,7 @@ class TensorOperation : public std::enable_shared_from_this<TensorOperation> {
/// \return shared pointer to the newly created TensorOp.
virtual std::shared_ptr<TensorOp> Build() = 0;
virtual bool ValidateParams() = 0;
virtual Status ValidateParams() = 0;
};
// Transform operations for performing data transformation.
@ -73,7 +74,7 @@ class OneHotOperation : public TensorOperation {
std::shared_ptr<TensorOp> Build() override;
bool ValidateParams() override;
Status ValidateParams() override;
private:
float num_classes_;
@ -87,7 +88,7 @@ class TypeCastOperation : public TensorOperation {
std::shared_ptr<TensorOp> Build() override;
bool ValidateParams() override;
Status ValidateParams() override;
private:
std::string data_type_;

57
mindspore/ccsrc/minddata/dataset/include/vision.h
Unescape View File

@ -17,10 +17,11 @@
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_VISION_H_
#define MINDSPORE_CCSRC_MINDDATA_DATASET_INCLUDE_VISION_H_
#include <vector>
#include <memory>
#include <vector>
#include "minddata/dataset/core/constants.h"
#include "minddata/dataset/include/transforms.h"
#include "minddata/dataset/util/status.h"
namespace mindspore {
namespace dataset {
@ -326,7 +327,7 @@ class CenterCropOperation : public TensorOperation {
std::shared_ptr<TensorOp> Build() override;
bool ValidateParams() override;
Status ValidateParams() override;
private:
std::vector<int32_t> size_;
@ -340,7 +341,7 @@ class CropOperation : public TensorOperation {
std::shared_ptr<TensorOp> Build() override;
bool ValidateParams() override;
Status ValidateParams() override;
private:
std::vector<int32_t> coordinates_;
@ -355,7 +356,7 @@ class CutMixBatchOperation : public TensorOperation {
std::shared_ptr<TensorOp> Build() override;
bool ValidateParams() override;
Status ValidateParams() override;
private:
float alpha_;
@ -371,7 +372,7 @@ class CutOutOperation : public TensorOperation {
std::shared_ptr<TensorOp> Build() override;
bool ValidateParams() override;
Status ValidateParams() override;
private:
int32_t length_;
@ -387,7 +388,7 @@ class DecodeOperation : public TensorOperation {
std::shared_ptr<TensorOp> Build() override;
bool ValidateParams() override;
Status ValidateParams() override;
private:
bool rgb_;
@ -399,7 +400,7 @@ class HwcToChwOperation : public TensorOperation {
std::shared_ptr<TensorOp> Build() override;
bool ValidateParams() override;
Status ValidateParams() override;
};
class MixUpBatchOperation : public TensorOperation {
@ -410,7 +411,7 @@ class MixUpBatchOperation : public TensorOperation {
std::shared_ptr<TensorOp> Build() override;
bool ValidateParams() override;
Status ValidateParams() override;
private:
float alpha_;
@ -424,7 +425,7 @@ class NormalizeOperation : public TensorOperation {
std::shared_ptr<TensorOp> Build() override;
bool ValidateParams() override;
Status ValidateParams() override;
private:
std::vector<float> mean_;
@ -440,7 +441,7 @@ class PadOperation : public TensorOperation {
std::shared_ptr<TensorOp> Build() override;
bool ValidateParams() override;
Status ValidateParams() override;
private:
std::vector<int32_t> padding_;
@ -460,7 +461,7 @@ class RandomAffineOperation : public TensorOperation {
std::shared_ptr<TensorOp> Build() override;
bool ValidateParams() override;
Status ValidateParams() override;
private:
std::vector<float_t> degrees_; // min_degree, max_degree
@ -479,7 +480,7 @@ class RandomColorOperation : public TensorOperation {
std::shared_ptr<TensorOp> Build() override;
bool ValidateParams() override;
Status ValidateParams() override;
private:
float t_lb_;
@ -495,7 +496,7 @@ class RandomColorAdjustOperation : public TensorOperation {
std::shared_ptr<TensorOp> Build() override;
bool ValidateParams() override;
Status ValidateParams() override;
private:
std::vector<float> brightness_;
@ -514,7 +515,7 @@ class RandomCropOperation : public TensorOperation {
std::shared_ptr<TensorOp> Build() override;
bool ValidateParams() override;
Status ValidateParams() override;
private:
std::vector<int32_t> size_;
@ -533,7 +534,7 @@ class RandomCropDecodeResizeOperation : public TensorOperation {
std::shared_ptr<TensorOp> Build() override;
bool ValidateParams() override;
Status ValidateParams() override;
private:
std::vector<int32_t> size_;
@ -551,7 +552,7 @@ class RandomHorizontalFlipOperation : public TensorOperation {
std::shared_ptr<TensorOp> Build() override;
bool ValidateParams() override;
Status ValidateParams() override;
private:
float probability_;
@ -565,7 +566,7 @@ class RandomPosterizeOperation : public TensorOperation {
std::shared_ptr<TensorOp> Build() override;
bool ValidateParams() override;
Status ValidateParams() override;
private:
std::vector<uint8_t> bit_range_;
@ -582,7 +583,7 @@ class RandomResizedCropOperation : public TensorOperation {
std::shared_ptr<TensorOp> Build() override;
bool ValidateParams() override;
Status ValidateParams() override;
private:
std::vector<int32_t> size_;
@ -601,7 +602,7 @@ class RandomRotationOperation : public TensorOperation {
std::shared_ptr<TensorOp> Build() override;
bool ValidateParams() override;
Status ValidateParams() override;
private:
std::vector<float> degrees_;
@ -619,7 +620,7 @@ class RandomSharpnessOperation : public TensorOperation {
std::shared_ptr<TensorOp> Build() override;
bool ValidateParams() override;
Status ValidateParams() override;
private:
std::vector<float> degrees_;
@ -633,7 +634,7 @@ class RandomSolarizeOperation : public TensorOperation {
std::shared_ptr<TensorOp> Build() override;
bool ValidateParams() override;
Status ValidateParams() override;
private:
std::vector<uint8_t> threshold_;
@ -647,7 +648,7 @@ class RandomVerticalFlipOperation : public TensorOperation {
std::shared_ptr<TensorOp> Build() override;
bool ValidateParams() override;
Status ValidateParams() override;
private:
float probability_;
@ -661,7 +662,7 @@ class RescaleOperation : public TensorOperation {
std::shared_ptr<TensorOp> Build() override;
bool ValidateParams() override;
Status ValidateParams() override;
private:
float rescale_;
@ -677,7 +678,7 @@ class ResizeOperation : public TensorOperation {
std::shared_ptr<TensorOp> Build() override;
bool ValidateParams() override;
Status ValidateParams() override;
private:
std::vector<int32_t> size_;
@ -692,7 +693,7 @@ class RgbaToBgrOperation : public TensorOperation {
std::shared_ptr<TensorOp> Build() override;
bool ValidateParams() override;
Status ValidateParams() override;
};
class RgbaToRgbOperation : public TensorOperation {
@ -703,7 +704,7 @@ class RgbaToRgbOperation : public TensorOperation {
std::shared_ptr<TensorOp> Build() override;
bool ValidateParams() override;
Status ValidateParams() override;
};
class SwapRedBlueOperation : public TensorOperation {
@ -714,7 +715,7 @@ class SwapRedBlueOperation : public TensorOperation {
std::shared_ptr<TensorOp> Build() override;
bool ValidateParams() override;
Status ValidateParams() override;
};
class UniformAugOperation : public TensorOperation {
@ -725,7 +726,7 @@ class UniformAugOperation : public TensorOperation {
std::shared_ptr<TensorOp> Build() override;
bool ValidateParams() override;
Status ValidateParams() override;
private:
std::vector<std::shared_ptr<TensorOperation>> transforms_;

Write Preview
Loading…
Cancel
Save