!9398 Change Build to return status

From: @alex-yuyue
Reviewed-by: @mikef
Signed-off-by:
pull/9398/MERGE
mindspore-ci-bot 4 years ago committed by Gitee
commit 73c91e05b1

@ -219,7 +219,7 @@ Status DataSchema::ColumnOrderLoad(nlohmann::json column_tree, const std::vector
// Find the column in the json document
auto column_info = column_tree.find(common::SafeCStr(curr_col_name));
if (column_info == column_tree.end()) {
RETURN_STATUS_UNEXPECTED("Failed to find column " + curr_col_name);
RETURN_STATUS_UNEXPECTED("Invalid data, failed to find column name: " + curr_col_name);
}
// At this point, columnInfo.value() is the subtree in the json document that contains
// all of the data for a given column. This data will formulate our schema column.
@ -246,7 +246,7 @@ Status DataSchema::ColumnOrderLoad(nlohmann::json column_tree, const std::vector
i++;
}
if (index == -1) {
RETURN_STATUS_UNEXPECTED("Failed to find column " + curr_col_name);
RETURN_STATUS_UNEXPECTED("Invalid data, failed to find column name: " + curr_col_name);
}
nlohmann::json column_child_tree = column_tree[index];
RETURN_IF_NOT_OK(ColumnLoad(column_child_tree, curr_col_name));

@ -91,27 +91,24 @@ Status BatchNode::ValidateParams() {
return Status::OK();
}
std::vector<std::shared_ptr<DatasetOp>> BatchNode::Build() {
// A vector containing shared pointer to the Dataset Ops that this object will create
std::vector<std::shared_ptr<DatasetOp>> node_ops;
Status BatchNode::Build(std::vector<std::shared_ptr<DatasetOp>> *node_ops) {
#ifdef ENABLE_PYTHON
// if col_order_ isn't empty, then a project node needs to be attached after batch node. (same as map)
// this means project_node needs to be the parent of batch_node. this means node_ops = [project_node, batch_node]
// this means project_node needs to be the parent of batch_node. this means *node_ops = [project_node, batch_node]
if (!col_order_.empty()) {
auto project_op = std::make_shared<ProjectOp>(col_order_);
node_ops.push_back(project_op);
node_ops->push_back(project_op);
}
node_ops.push_back(std::make_shared<BatchOp>(batch_size_, drop_remainder_, pad_, connector_que_size_, num_workers_,
in_col_names_, out_col_names_, batch_size_func_, batch_map_func_,
pad_map_));
node_ops->push_back(std::make_shared<BatchOp>(batch_size_, drop_remainder_, pad_, connector_que_size_, num_workers_,
in_col_names_, out_col_names_, batch_size_func_, batch_map_func_,
pad_map_));
#else
node_ops.push_back(std::make_shared<BatchOp>(batch_size_, drop_remainder_, pad_, connector_que_size_, num_workers_,
in_col_names_, pad_map_));
node_ops->push_back(std::make_shared<BatchOp>(batch_size_, drop_remainder_, pad_, connector_que_size_, num_workers_,
in_col_names_, pad_map_));
#endif
return node_ops;
return Status::OK();
}
// Get Dataset size

@ -57,8 +57,9 @@ class BatchNode : public DatasetNode {
std::shared_ptr<DatasetNode> Copy() override;
/// \brief a base class override function to create the required runtime dataset op objects for this class
/// \return The list of shared pointers to the newly created DatasetOps
std::vector<std::shared_ptr<DatasetOp>> Build() override;
/// \param node_ops - A vector containing shared pointer to the Dataset Ops that this object will create
/// \return Status Status::OK() if build successfully
Status Build(std::vector<std::shared_ptr<DatasetOp>> *node_ops) override;
/// \brief Parameters validation
/// \return Status Status::OK() if all the parameters are valid

@ -54,17 +54,15 @@ void BucketBatchByLengthNode::Print(std::ostream &out) const {
out << Name() + "(columns:" + PrintColumns(column_names_) + ",...)";
}
std::vector<std::shared_ptr<DatasetOp>> BucketBatchByLengthNode::Build() {
// A vector containing shared pointer to the Dataset Ops that this object will create
std::vector<std::shared_ptr<DatasetOp>> node_ops;
Status BucketBatchByLengthNode::Build(std::vector<std::shared_ptr<DatasetOp>> *node_ops) {
bucket_boundaries_.insert(bucket_boundaries_.begin(), 0);
node_ops.push_back(std::make_shared<BucketBatchByLengthOp>(
node_ops->push_back(std::make_shared<BucketBatchByLengthOp>(
column_names_, bucket_boundaries_, bucket_batch_sizes_, element_length_function_, pad_info_,
pad_to_bucket_boundary_, drop_remainder_, connector_que_size_));
if (bucket_boundaries_[0] == 0) {
bucket_boundaries_.erase(bucket_boundaries_.begin());
}
return node_ops;
return Status::OK();
}
Status BucketBatchByLengthNode::ValidateParams() {

@ -53,8 +53,9 @@ class BucketBatchByLengthNode : public DatasetNode {
std::shared_ptr<DatasetNode> Copy() override;
/// \brief a base class override function to create the required runtime dataset op objects for this class
/// \return The list of shared pointers to the newly created DatasetOps
std::vector<std::shared_ptr<DatasetOp>> Build() override;
/// \param node_ops - A vector containing shared pointer to the Dataset Ops that this object will create
/// \return Status Status::OK() if build successfully
Status Build(std::vector<std::shared_ptr<DatasetOp>> *node_ops) override;
/// \brief Parameters validation
/// \return Status Status::OK() if all the parameters are valid

@ -54,15 +54,12 @@ void BuildSentenceVocabNode::Print(std::ostream &out) const {
}
// Function to build BuildSentenceVocabNode
std::vector<std::shared_ptr<DatasetOp>> BuildSentenceVocabNode::Build() {
// A vector containing shared pointer to the Dataset Ops that this object will create
std::vector<std::shared_ptr<DatasetOp>> node_ops;
Status BuildSentenceVocabNode::Build(std::vector<std::shared_ptr<DatasetOp>> *node_ops) {
std::shared_ptr<BuildSentencePieceVocabOp> build_sentence_piece_vocab_op;
build_sentence_piece_vocab_op = std::make_shared<BuildSentencePieceVocabOp>(
vocab_, col_names_, vocab_size_, character_coverage_, model_type_, params_, connector_que_size_);
node_ops.push_back(build_sentence_piece_vocab_op);
return node_ops;
node_ops->push_back(build_sentence_piece_vocab_op);
return Status::OK();
}
Status BuildSentenceVocabNode::ValidateParams() {

@ -51,8 +51,9 @@ class BuildSentenceVocabNode : public DatasetNode {
std::shared_ptr<DatasetNode> Copy() override;
/// \brief a base class override function to create the required runtime dataset op objects for this class
/// \return The list of shared pointers to the newly created DatasetOps
std::vector<std::shared_ptr<DatasetOp>> Build() override;
/// \param node_ops - A vector containing shared pointer to the Dataset Ops that this object will create
/// \return Status Status::OK() if build successfully
Status Build(std::vector<std::shared_ptr<DatasetOp>> *node_ops) override;
/// \brief Parameters validation
/// \return Status Status::OK() if all the parameters are valid

@ -50,15 +50,12 @@ void BuildVocabNode::Print(std::ostream &out) const {
}
// Function to build BuildVocabNode
std::vector<std::shared_ptr<DatasetOp>> BuildVocabNode::Build() {
// A vector containing shared pointer to the Dataset Ops that this object will create
std::vector<std::shared_ptr<DatasetOp>> node_ops;
Status BuildVocabNode::Build(std::vector<std::shared_ptr<DatasetOp>> *node_ops) {
std::shared_ptr<BuildVocabOp> build_vocab_op;
build_vocab_op = std::make_shared<BuildVocabOp>(vocab_, columns_, freq_range_, top_k_, special_tokens_,
special_first_, num_workers_, connector_que_size_);
node_ops.push_back(build_vocab_op);
return node_ops;
node_ops->push_back(build_vocab_op);
return Status::OK();
}
Status BuildVocabNode::ValidateParams() {

@ -50,8 +50,9 @@ class BuildVocabNode : public DatasetNode {
std::shared_ptr<DatasetNode> Copy() override;
/// \brief a base class override function to create the required runtime dataset op objects for this class
/// \return The list of shared pointers to the newly created DatasetOps
std::vector<std::shared_ptr<DatasetOp>> Build() override;
/// \param node_ops - A vector containing shared pointer to the Dataset Ops that this object will create
/// \return Status Status::OK() if build successfully
Status Build(std::vector<std::shared_ptr<DatasetOp>> *node_ops) override;
/// \brief Parameters validation
/// \return Status Status::OK() if all the parameters are valid

@ -69,17 +69,15 @@ Status ConcatNode::ValidateParams() {
return Status::OK();
}
std::vector<std::shared_ptr<DatasetOp>> ConcatNode::Build() {
// A vector containing shared pointer to the Dataset Ops that this object will create
std::vector<std::shared_ptr<DatasetOp>> node_ops;
Status ConcatNode::Build(std::vector<std::shared_ptr<DatasetOp>> *node_ops) {
if (children_flag_and_nums_.empty() || children_start_end_index_.empty()) {
node_ops.push_back(std::make_shared<ConcatOp>(connector_que_size_));
node_ops->push_back(std::make_shared<ConcatOp>(connector_que_size_));
} else {
node_ops.push_back(std::make_shared<ConcatOp>(connector_que_size_, sampler_->Build(), children_flag_and_nums_,
children_start_end_index_));
node_ops->push_back(std::make_shared<ConcatOp>(connector_que_size_, sampler_->Build(), children_flag_and_nums_,
children_start_end_index_));
}
return node_ops;
return Status::OK();
}
// Visitor accepting method for NodePass

@ -51,8 +51,9 @@ class ConcatNode : public DatasetNode {
std::shared_ptr<DatasetNode> Copy() override;
/// \brief a base class override function to create the required runtime dataset op objects for this class
/// \return The list of shared pointers to the newly created DatasetOps
std::vector<std::shared_ptr<DatasetOp>> Build() override;
/// \param node_ops - A vector containing shared pointer to the Dataset Ops that this object will create
/// \return Status Status::OK() if build successfully
Status Build(std::vector<std::shared_ptr<DatasetOp>> *node_ops) override;
/// \brief Parameters validation
/// \return Status Status::OK() if all the parameters are valid

@ -60,7 +60,7 @@ Status AddShuffleOp(int64_t num_files, int64_t num_devices, int64_t num_rows, in
int32_t connector_que_size, int32_t rows_per_buffer, std::shared_ptr<DatasetOp> *shuffle_op) {
std::shared_ptr<ShuffleOp> new_shuffle_op = nullptr;
int64_t shuffle_size = 0;
RETURN_EMPTY_IF_ERROR(ComputeShuffleSize(num_files, num_devices, num_rows, total_rows, &shuffle_size));
RETURN_IF_NOT_OK(ComputeShuffleSize(num_files, num_devices, num_rows, total_rows, &shuffle_size));
MS_LOG(INFO) << "Dataset::AddShuffleOp - num_rows: " << num_rows << ", shuffle_size: " << shuffle_size;
// Add the shuffle op
*shuffle_op = std::make_shared<ShuffleOp>(shuffle_size, GetSeed(), connector_que_size, true, rows_per_buffer);

@ -35,15 +35,6 @@ class SamplerObj;
class NodePass;
class DatasetSizeGetter;
#define RETURN_EMPTY_IF_ERROR(_s) \
do { \
Status __rc = (_s); \
if (__rc.IsError()) { \
MS_LOG(ERROR) << __rc; \
return {}; \
} \
} while (false)
// Names for non-leaf IR node
constexpr char kBatchNode[] = "Batch";
constexpr char kBucketBatchByLengthNode[] = "BucketBatchByLength";
@ -148,7 +139,7 @@ class DatasetNode : public std::enable_shared_from_this<DatasetNode> {
/// \brief << Stream output operator overload
/// \notes This allows you to write the debug print info using stream operators
/// \param out - reference to the output stream being overloaded
/// \param dO - reference to the DatasetOp to display
/// \param node - reference to the DatasetNode to display
/// \return - the output stream must be returned
friend std::ostream &operator<<(std::ostream &out, const DatasetNode &node) {
node.PrintTree(out);
@ -160,8 +151,9 @@ class DatasetNode : public std::enable_shared_from_this<DatasetNode> {
std::shared_ptr<DatasetNode> DeepCopy();
/// \brief Pure virtual function to convert a DatasetNode class into a runtime dataset object
/// \return The list of shared pointers to the newly created DatasetOps
virtual std::vector<std::shared_ptr<DatasetOp>> Build() = 0;
/// \param node_ops - A vector containing shared pointer to the Dataset Ops that this object will create
/// \return Status Status::OK() if build successfully
virtual Status Build(std::vector<std::shared_ptr<DatasetOp>> *node_ops) = 0;
/// \brief Pure virtual function for derived class to implement parameters validation
/// \return Status Status::OK() if all the parameters are valid
@ -229,10 +221,6 @@ class DatasetNode : public std::enable_shared_from_this<DatasetNode> {
/// \return Status of the node visit
virtual Status AcceptAfter(NodePass *p, bool *modified);
/// \brief Method to get status from Node.Build()
/// \notes Remove me after changing return val of Build()
Status BuildStatus() { return build_status; }
virtual bool IsSizeDefined() { return true; }
protected:
@ -244,7 +232,6 @@ class DatasetNode : public std::enable_shared_from_this<DatasetNode> {
int32_t rows_per_buffer_;
int32_t connector_que_size_;
int32_t worker_connector_size_;
Status build_status; // remove me after changing return val of Build()
std::string PrintColumns(const std::vector<std::string> &columns) const;
Status AddCacheOp(std::vector<std::shared_ptr<DatasetOp>> *node_ops);
void PrintNode(std::ostream &out, int *level) const;

@ -40,11 +40,9 @@ std::shared_ptr<DatasetNode> EpochCtrlNode::Copy() {
void EpochCtrlNode::Print(std::ostream &out) const { out << Name() + "(epoch:" + std::to_string(num_epochs_) + ")"; }
// Function to build the EpochCtrlOp
std::vector<std::shared_ptr<DatasetOp>> EpochCtrlNode::Build() {
// A dummy vector
std::vector<std::shared_ptr<DatasetOp>> node_ops;
node_ops.push_back(std::make_shared<EpochCtrlOp>(num_epochs_));
return node_ops;
Status EpochCtrlNode::Build(std::vector<std::shared_ptr<DatasetOp>> *node_ops) {
node_ops->push_back(std::make_shared<EpochCtrlOp>(num_epochs_));
return Status::OK();
}
// Function to validate the parameters for EpochCtrlNode

@ -47,8 +47,9 @@ class EpochCtrlNode : public DatasetNode {
std::shared_ptr<DatasetNode> Copy() override;
/// \brief a base class override function to create the required runtime dataset op objects for this class
/// \return shared pointer to the list of newly created DatasetOps
std::vector<std::shared_ptr<DatasetOp>> Build() override;
/// \param node_ops - A vector containing shared pointer to the Dataset Ops that this object will create
/// \return Status Status::OK() if build successfully
Status Build(std::vector<std::shared_ptr<DatasetOp>> *node_ops) override;
/// \brief Parameters validation
/// \return Status Status::OK() if all the parameters are valid

@ -43,12 +43,9 @@ void FilterNode::Print(std::ostream &out) const {
out << Name() + "(<predicate>," + "input_cols:" + PrintColumns(input_columns_) + ")";
}
std::vector<std::shared_ptr<DatasetOp>> FilterNode::Build() {
// A vector containing shared pointer to the Dataset Ops that this object will create
std::vector<std::shared_ptr<DatasetOp>> node_ops;
node_ops.push_back(std::make_shared<FilterOp>(input_columns_, num_workers_, connector_que_size_, predicate_));
return node_ops;
Status FilterNode::Build(std::vector<std::shared_ptr<DatasetOp>> *node_ops) {
node_ops->push_back(std::make_shared<FilterOp>(input_columns_, num_workers_, connector_que_size_, predicate_));
return Status::OK();
}
Status FilterNode::ValidateParams() {

@ -48,8 +48,9 @@ class FilterNode : public DatasetNode {
std::shared_ptr<DatasetNode> Copy() override;
/// \brief a base class override function to create the required runtime dataset op objects for this class
/// \return The list of shared pointers to the newly created DatasetOps
std::vector<std::shared_ptr<DatasetOp>> Build() override;
/// \param node_ops - A vector containing shared pointer to the Dataset Ops that this object will create
/// \return Status Status::OK() if build successfully
Status Build(std::vector<std::shared_ptr<DatasetOp>> *node_ops) override;
/// \brief Parameters validation
/// \return Status Status::OK() if all the parameters are valid

@ -52,10 +52,7 @@ void MapNode::Print(std::ostream &out) const {
",<project_cols>" + ",...)";
}
std::vector<std::shared_ptr<DatasetOp>> MapNode::Build() {
// A vector containing shared pointer to the Dataset Ops that this object will create
std::vector<std::shared_ptr<DatasetOp>> node_ops;
Status MapNode::Build(std::vector<std::shared_ptr<DatasetOp>> *node_ops) {
std::vector<std::shared_ptr<TensorOp>> tensor_ops;
// Build tensorOp from tensorOperation vector
@ -74,13 +71,12 @@ std::vector<std::shared_ptr<DatasetOp>> MapNode::Build() {
if (!project_columns_.empty()) {
auto project_op = std::make_shared<ProjectOp>(project_columns_);
node_ops.push_back(project_op);
node_ops->push_back(project_op);
}
build_status = AddCacheOp(&node_ops); // remove me after changing return val of Build()
RETURN_EMPTY_IF_ERROR(build_status);
RETURN_IF_NOT_OK(AddCacheOp(node_ops));
node_ops.push_back(map_op);
return node_ops;
node_ops->push_back(map_op);
return Status::OK();
}
Status MapNode::ValidateParams() {

@ -50,8 +50,9 @@ class MapNode : public DatasetNode {
std::shared_ptr<DatasetNode> Copy() override;
/// \brief a base class override function to create the required runtime dataset op objects for this class
/// \return The list of shared pointers to the newly created DatasetOps
std::vector<std::shared_ptr<DatasetOp>> Build() override;
/// \param node_ops - A vector containing shared pointer to the Dataset Ops that this object will create
/// \return Status Status::OK() if build successfully
Status Build(std::vector<std::shared_ptr<DatasetOp>> *node_ops) override;
/// \brief Parameters validation
/// \return Status Status::OK() if all the parameters are valid

@ -51,12 +51,9 @@ Status ProjectNode::ValidateParams() {
return Status::OK();
}
std::vector<std::shared_ptr<DatasetOp>> ProjectNode::Build() {
// A vector containing shared pointer to the Dataset Ops that this object will create
std::vector<std::shared_ptr<DatasetOp>> node_ops;
node_ops.push_back(std::make_shared<ProjectOp>(columns_));
return node_ops;
Status ProjectNode::Build(std::vector<std::shared_ptr<DatasetOp>> *node_ops) {
node_ops->push_back(std::make_shared<ProjectOp>(columns_));
return Status::OK();
}
} // namespace dataset

@ -47,8 +47,9 @@ class ProjectNode : public DatasetNode {
std::shared_ptr<DatasetNode> Copy() override;
/// \brief a base class override function to create the required runtime dataset op objects for this class
/// \return The list of shared pointers to the newly created DatasetOps
std::vector<std::shared_ptr<DatasetOp>> Build() override;
/// \param node_ops - A vector containing shared pointer to the Dataset Ops that this object will create
/// \return Status Status::OK() if build successfully
Status Build(std::vector<std::shared_ptr<DatasetOp>> *node_ops) override;
/// \brief Parameters validation
/// \return Status Status::OK() if all the parameters are valid

@ -56,12 +56,9 @@ Status RenameNode::ValidateParams() {
return Status::OK();
}
std::vector<std::shared_ptr<DatasetOp>> RenameNode::Build() {
// A vector containing shared pointer to the Dataset Ops that this object will create
std::vector<std::shared_ptr<DatasetOp>> node_ops;
node_ops.push_back(std::make_shared<RenameOp>(input_columns_, output_columns_, connector_que_size_));
return node_ops;
Status RenameNode::Build(std::vector<std::shared_ptr<DatasetOp>> *node_ops) {
node_ops->push_back(std::make_shared<RenameOp>(input_columns_, output_columns_, connector_que_size_));
return Status::OK();
}
} // namespace dataset

@ -48,8 +48,9 @@ class RenameNode : public DatasetNode {
std::shared_ptr<DatasetNode> Copy() override;
/// \brief a base class override function to create the required runtime dataset op objects for this class
/// \return The list of shared pointers to the newly created DatasetOps
std::vector<std::shared_ptr<DatasetOp>> Build() override;
/// \param node_ops - A vector containing shared pointer to the Dataset Ops that this object will create
/// \return Status Status::OK() if build successfully
Status Build(std::vector<std::shared_ptr<DatasetOp>> *node_ops) override;
/// \brief Parameters validation
/// \return Status Status::OK() if all the parameters are valid

@ -37,12 +37,9 @@ std::shared_ptr<DatasetNode> RepeatNode::Copy() {
void RepeatNode::Print(std::ostream &out) const { out << Name() + "(count:" + std::to_string(repeat_count_) + ")"; }
std::vector<std::shared_ptr<DatasetOp>> RepeatNode::Build() {
// A vector containing shared pointer to the Dataset Ops that this object will create
std::vector<std::shared_ptr<DatasetOp>> node_ops;
node_ops.push_back(std::make_shared<RepeatOp>(repeat_count_));
return node_ops;
Status RepeatNode::Build(std::vector<std::shared_ptr<DatasetOp>> *node_ops) {
node_ops->push_back(std::make_shared<RepeatOp>(repeat_count_));
return Status::OK();
}
Status RepeatNode::ValidateParams() {

@ -49,8 +49,9 @@ class RepeatNode : public DatasetNode {
std::shared_ptr<DatasetNode> Copy() override;
/// \brief a base class override function to create the required runtime dataset op objects for this class
/// \return The list of shared pointers to the newly created DatasetOps
std::vector<std::shared_ptr<DatasetOp>> Build() override;
/// \param node_ops - A vector containing shared pointer to the Dataset Ops that this object will create
/// \return Status Status::OK() if build successfully
Status Build(std::vector<std::shared_ptr<DatasetOp>> *node_ops) override;
/// \brief Parameters validation
/// \return Status Status::OK() if all the parameters are valid

Some files were not shown because too many files have changed in this diff Show More

Loading…
Cancel
Save