m53297601
/
mindspore
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Make Parse() as protected

Browse Source
pull/12762/head
TinaMengtingZhang 4 years ago
parent a5af03f8ca
commit ccefa8b32e
9 changed files with 173 additions and 10 deletions
Show Stats Download Patch File Download Diff File

8
mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/map_node.cc
Unescape View File

@ -104,7 +104,13 @@ Status MapNode::ValidateParams() {
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
for (const auto &op : operations_) {
RETURN_IF_NOT_OK(op->ValidateParams());
if (op == nullptr) {
std::string err_msg = "MapNode: operation must not be nullptr.";
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
} else {
RETURN_IF_NOT_OK(op->ValidateParams());
}
}
if (!input_columns_.empty()) {
RETURN_IF_NOT_OK(ValidateDatasetColumnParam("MapNode", "input_columns", input_columns_));

36
mindspore/ccsrc/minddata/dataset/include/datasets.h
Unescape View File

@ -281,8 +281,8 @@ class Dataset : public std::enable_shared_from_this<Dataset> {
/// \brief Function to create a MapDataset
/// \notes Applies each operation in operations to this dataset
/// \param[in] operations Vector of operations to be applied on the dataset. Operations are
/// applied in the order they appear in this list
/// \param[in] operations Vector of raw pointers to TensorTransform objects to be applied on the dataset. Operations
/// are applied in the order they appear in this list
/// \param[in] input_columns Vector of the names of the columns that will be passed to the first
/// operation as input. The size of this list must match the number of
/// input columns expected by the first operator. The default input_columns
@ -309,6 +309,22 @@ class Dataset : public std::enable_shared_from_this<Dataset> {
project_columns, cache, callbacks);
}
/// \brief Function to create a MapDataset
/// \notes Applies each operation in operations to this dataset
/// \param[in] operations Vector of shared pointers to TensorTransform objects to be applied on the dataset.
/// Operations are applied in the order they appear in this list
/// \param[in] input_columns Vector of the names of the columns that will be passed to the first
/// operation as input. The size of this list must match the number of
/// input columns expected by the first operator. The default input_columns
/// is the first column
/// \param[in] output_columns Vector of names assigned to the columns outputted by the last operation
/// This parameter is mandatory if len(input_columns) != len(output_columns)
/// The size of this list must match the number of output columns of the
/// last operation. The default output_columns will have the same
/// name as the input columns, i.e., the columns will be replaced
/// \param[in] project_columns A list of column names to project
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
/// \return Shared pointer to the current MapDataset
std::shared_ptr<MapDataset> Map(std::vector<std::shared_ptr<TensorTransform>> operations,
const std::vector<std::string> &input_columns = {},
const std::vector<std::string> &output_columns = {},
@ -324,6 +340,22 @@ class Dataset : public std::enable_shared_from_this<Dataset> {
project_columns, cache, callbacks);
}
/// \brief Function to create a MapDataset
/// \notes Applies each operation in operations to this dataset
/// \param[in] operations Vector of TensorTransform objects to be applied on the dataset. Operations are applied in
/// the order they appear in this list
/// \param[in] input_columns Vector of the names of the columns that will be passed to the first
/// operation as input. The size of this list must match the number of
/// input columns expected by the first operator. The default input_columns
/// is the first column
/// \param[in] output_columns Vector of names assigned to the columns outputted by the last operation
/// This parameter is mandatory if len(input_columns) != len(output_columns)
/// The size of this list must match the number of output columns of the
/// last operation. The default output_columns will have the same
/// name as the input columns, i.e., the columns will be replaced
/// \param[in] project_columns A list of column names to project
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
/// \return Shared pointer to the current MapDataset
std::shared_ptr<MapDataset> Map(const std::vector<std::reference_wrapper<TensorTransform>> operations,
const std::vector<std::string> &input_columns = {},
const std::vector<std::string> &output_columns = {},

20
mindspore/ccsrc/minddata/dataset/include/text.h
Unescape View File

@ -59,6 +59,7 @@ class BasicTokenizer : public TensorTransform {
/// \brief Destructor
~BasicTokenizer() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -99,6 +100,7 @@ class BertTokenizer : public TensorTransform {
/// \brief Destructor
~BertTokenizer() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -125,6 +127,7 @@ class CaseFold : public TensorTransform {
/// \brief Destructor
~CaseFold() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
//// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -151,12 +154,13 @@ class JiebaTokenizer : public TensorTransform {
/// \brief Destructor
~JiebaTokenizer() = default;
Status AddWord(const std::string &word, int64_t freq = 0);
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
Status AddWord(const std::string &word, int64_t freq = 0);
private:
std::string hmm_path_;
std::string mp_path_;
@ -180,6 +184,7 @@ class Lookup : public TensorTransform {
/// \brief Destructor
~Lookup() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -208,6 +213,7 @@ class Ngram : public TensorTransform {
/// \brief Destructor
~Ngram() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -238,6 +244,7 @@ class NormalizeUTF8 : public TensorTransform {
/// \brief Destructor
~NormalizeUTF8() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -259,6 +266,7 @@ class RegexReplace : public TensorTransform {
/// \brief Destructor
~RegexReplace() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -283,6 +291,7 @@ class RegexTokenizer : public TensorTransform {
/// \brief Destructor
~RegexTokenizer() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -311,6 +320,7 @@ class SentencePieceTokenizer : public TensorTransform {
/// \brief Destructor
~SentencePieceTokenizer() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -335,6 +345,7 @@ class SlidingWindow : public TensorTransform {
/// \brief Destructor
~SlidingWindow() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -358,6 +369,7 @@ class ToNumber : public TensorTransform {
/// \brief Destructor
~ToNumber() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -376,6 +388,7 @@ class TruncateSequencePair : public TensorTransform {
/// \brief Destructor
~TruncateSequencePair() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -394,6 +407,7 @@ class UnicodeCharTokenizer : public TensorTransform {
/// \brief Destructor
~UnicodeCharTokenizer() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -414,6 +428,7 @@ class UnicodeScriptTokenizer : public TensorTransform {
/// \brief Destructor
~UnicodeScriptTokenizer() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -433,6 +448,7 @@ class WhitespaceTokenizer : public TensorTransform {
/// \brief Destructor
~WhitespaceTokenizer() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;

50
mindspore/ccsrc/minddata/dataset/include/transforms.h
Unescape View File

@ -30,10 +30,32 @@ namespace dataset {
class TensorOperation;
// We need the following two groups of forward declaration to friend the class in class TensorTransform.
namespace transforms {
class Compose;
class RandomApply;
class RandomChoice;
} // namespace transforms
namespace vision {
class BoundingBoxAugment;
class RandomSelectSubpolicy;
class UniformAugment;
} // namespace vision
// Abstract class to represent a tensor transform operation in the data pipeline.
/// \class TensorTransform transforms.h
/// \brief A base class to represent a tensor transform operation in the data pipeline.
class TensorTransform : public std::enable_shared_from_this<TensorTransform> {
friend class Dataset;
friend class Execute;
friend class transforms::Compose;
friend class transforms::RandomApply;
friend class transforms::RandomChoice;
friend class vision::BoundingBoxAugment;
friend class vision::RandomSelectSubpolicy;
friend class vision::UniformAugment;
public:
/// \brief Constructor
TensorTransform() {}
@ -41,6 +63,7 @@ class TensorTransform : public std::enable_shared_from_this<TensorTransform> {
/// \brief Destructor
~TensorTransform() = default;
protected:
/// \brief Pure virtual function to convert a TensorTransform class into a IR TensorOperation object.
/// \return shared pointer to the newly created TensorOperation.
virtual std::shared_ptr<TensorOperation> Parse() = 0;
@ -59,14 +82,19 @@ namespace transforms {
class Compose : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] transforms A vector of transformations to be applied.
/// \param[in] transforms A vector of raw pointers to TensorTransform objects to be applied.
explicit Compose(const std::vector<TensorTransform *> &transforms);
/// \brief Constructor.
/// \param[in] transforms A vector of shared pointers to TensorTransform objects to be applied.
explicit Compose(const std::vector<std::shared_ptr<TensorTransform>> &transforms);
/// \brief Constructor.
/// \param[in] transforms A vector of TensorTransform objects to be applied.
explicit Compose(const std::vector<std::reference_wrapper<TensorTransform>> &transforms);
/// \brief Destructor
~Compose() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -86,6 +114,7 @@ class Duplicate : public TensorTransform {
/// \brief Destructor
~Duplicate() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -102,6 +131,7 @@ class OneHot : public TensorTransform {
/// \brief Destructor
~OneHot() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -115,15 +145,22 @@ class OneHot : public TensorTransform {
class RandomApply : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] transforms A vector of transformations to be applied.
/// \param[in] transforms A vector of raw pointers to TensorTransform objects to be applied.
/// \param[in] prob The probability to apply the transformation list (default=0.5)
explicit RandomApply(const std::vector<TensorTransform *> &transforms, double prob = 0.5);
/// \brief Constructor.
/// \param[in] transforms A vector of shared pointers to TensorTransform objects to be applied.
/// \param[in] prob The probability to apply the transformation list (default=0.5)
explicit RandomApply(const std::vector<std::shared_ptr<TensorTransform>> &transforms, double prob = 0.5);
/// \brief Constructor.
/// \param[in] transforms A vector of TensorTransform objects to be applied.
/// \param[in] prob The probability to apply the transformation list (default=0.5)
explicit RandomApply(const std::vector<std::reference_wrapper<TensorTransform>> &transforms, double prob = 0.5);
/// \brief Destructor
~RandomApply() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -138,14 +175,19 @@ class RandomApply : public TensorTransform {
class RandomChoice : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] transforms A vector of transformations to be chosen from to apply.
/// \param[in] transforms A vector of raw pointers to TensorTransform objects to be applied.
explicit RandomChoice(const std::vector<TensorTransform *> &transforms);
/// \brief Constructor.
/// \param[in] transforms A vector of shared pointers to TensorTransform objects to be applied.
explicit RandomChoice(const std::vector<std::shared_ptr<TensorTransform>> &transforms);
/// \brief Constructor.
/// \param[in] transforms A vector of TensorTransform objects to be applied.
explicit RandomChoice(const std::vector<std::reference_wrapper<TensorTransform>> &transforms);
/// \brief Destructor
~RandomChoice() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -165,6 +207,7 @@ class TypeCast : public TensorTransform {
/// \brief Destructor
~TypeCast() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -184,6 +227,7 @@ class Unique : public TensorTransform {
/// \brief Destructor
~Unique() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;

36
mindspore/ccsrc/minddata/dataset/include/vision.h
View File

File diff suppressed because it is too large Load Diff

3
mindspore/ccsrc/minddata/dataset/include/vision_ascend.h
Unescape View File

@ -47,6 +47,7 @@ class DvppDecodeResizeJpeg : public TensorTransform {
/// \brief Destructor.
~DvppDecodeResizeJpeg() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -67,6 +68,7 @@ class DvppDecodeResizeCropJpeg : public TensorTransform {
/// \brief Destructor.
~DvppDecodeResizeCropJpeg() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -86,6 +88,7 @@ class DvppDecodePng : public TensorTransform {
/// \brief Destructor.
~DvppDecodePng() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;

6
mindspore/ccsrc/minddata/dataset/include/vision_lite.h
Unescape View File

@ -83,6 +83,7 @@ class CenterCrop : public TensorTransform {
/// \brief Destructor.
~CenterCrop() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -107,6 +108,7 @@ class Crop : public TensorTransform {
/// \brief Destructor.
~Crop() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -127,6 +129,7 @@ class Decode : public TensorTransform {
/// \brief Destructor.
~Decode() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -151,6 +154,7 @@ class Normalize : public TensorTransform {
/// \brief Destructor.
~Normalize() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -218,6 +222,7 @@ class Resize : public TensorTransform {
/// \brief Destructor.
~Resize() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
@ -239,6 +244,7 @@ class Rotate : public TensorTransform {
/// \brief Destructor.
~Rotate() = default;
protected:
/// \brief Function to convert TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;

19
tests/ut/cpp/dataset/c_api_dataset_ops_test.cc
Unescape View File

@ -997,6 +997,25 @@ TEST_F(MindDataTestPipeline, TestMapDuplicateColumnFail) {
EXPECT_EQ(iter3, nullptr);
}
TEST_F(MindDataTestPipeline, TestMapNullOperation) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestMapNullOperation.";
// Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
EXPECT_NE(ds, nullptr);
// Create a Map operation on ds
std::shared_ptr<TensorTransform> operation = nullptr;
auto ds1 = ds->Map({operation}, {"image"}, {}, {});
EXPECT_NE(ds1, nullptr);
// Create an iterator over the result of the above dataset
std::shared_ptr<Iterator> iter1 = ds1->CreateIterator();
// Expect failure: Operation is nullptr
EXPECT_EQ(iter1, nullptr);
}
TEST_F(MindDataTestPipeline, TestProjectMapAutoInjection) {
MS_LOG(INFO) << "Doing MindDataTestPipeline.TestProjectMapAutoInjection";

5
tests/ut/cpp/dataset/optimization_pass_test.cc
Unescape View File

@ -91,8 +91,9 @@ TEST_F(MindDataTestOptimizationPass, MindDataTestTensorFusionPassPreBuiltTensorO
MS_LOG(INFO) << "Doing MindDataTestOptimizationPass-MindDataTestTensorFusionPassPreBuiltTensorOperation.";
std::string folder_path = datasets_root_path_ + "/testPK/data/";
// make prebuilt tensor operation
auto decode = std::make_shared<transforms::PreBuiltOperation>(vision::Decode().Parse()->Build());
auto resize = std::make_shared<transforms::PreBuiltOperation>(vision::RandomResizedCrop({100}).Parse()->Build());
auto decode = std::make_shared<transforms::PreBuiltOperation>(vision::DecodeOperation(true).Build());
auto resize = std::make_shared<transforms::PreBuiltOperation>(
vision::RandomResizedCropOperation({100}, {0.5}, {0.1}, InterpolationMode::kNearestNeighbour, 5).Build());
std::vector<std::shared_ptr<TensorOperation>> op_list = {decode, resize};
std::vector<std::string> op_name = {"image"};
std::shared_ptr<DatasetNode> root = ImageFolder(folder_path, false)->IRNode();

Write Preview
Loading…
Cancel
Save