m53297601
/
mindspore
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

!11598 [MD] add common validation functions for transform

Browse Source 
From: @luoyang42
Reviewed-by: 
Signed-off-by:
pull/11598/MERGE
mindspore-ci-bot 4 years ago committed by Gitee
parent d429ea3f7d 1893ae8662
commit f980561fe1
4 changed files with 276 additions and 536 deletions
Show Stats Download Patch File Download Diff File

166
mindspore/ccsrc/minddata/dataset/api/transforms.cc
Unescape View File

@ -31,85 +31,169 @@ namespace mindspore {
namespace dataset { namespace dataset {
/* ####################################### Validator Functions ############################################ */ /* ####################################### Validator Functions ############################################ */
Status ValidateVectorFillvalue(const std::string &transform_name, const std::vector<uint8_t> &fill_value) { Status ValidateProbability(const std::string &op_name, const float probability) {
if (probability < 0.0 || probability > 1.0) {
std::string err_msg = op_name + ": probability must be between 0.0 and 1.0, got: " + std::to_string(probability);
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
return Status::OK();
}
Status ValidateIntScalarPositive(const std::string &op_name, const std::string &scalar_name, int32_t scalar) {
RETURN_IF_NOT_OK(ValidateScalar(op_name, scalar_name, scalar, {0}, true));
return Status::OK();
}
Status ValidateFloatScalarPositive(const std::string &op_name, const std::string &scalar_name, float scalar) {
RETURN_IF_NOT_OK(ValidateScalar(op_name, scalar_name, scalar, {0}, true));
return Status::OK();
}
Status ValidateVectorFillvalue(const std::string &op_name, const std::vector<uint8_t> &fill_value) {
if (fill_value.empty() || (fill_value.size() != 1 && fill_value.size() != 3)) { if (fill_value.empty() || (fill_value.size() != 1 && fill_value.size() != 3)) {
std::string err_msg = std::string err_msg =
transform_name + ": fill_value vector has incorrect size: " + std::to_string(fill_value.size()); op_name + ": fill_value expecting size 1 or 3, got fill_value.size(): " + std::to_string(fill_value.size());
MS_LOG(ERROR) << err_msg; MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg); RETURN_STATUS_SYNTAX_ERROR(err_msg);
} }
for (uint8_t single_fill_value : fill_value) { // Note that fill_value need to be in range [0, 255],
if (single_fill_value > 255) { // but we omit the check since its type is uint8_t
std::string err_msg = return Status::OK();
transform_name + ": fill_value has to be between 0 and 255, got:" + std::to_string(single_fill_value); }
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg); Status ValidateVectorColorAttribute(const std::string &op_name, const std::string &attr_name,
} const std::vector<float> &attr, const std::vector<float> &range) {
if (attr.empty() || attr.size() > 2) {
std::string err_msg = op_name + ":" + attr_name + " expecting size 1 or 2, but got: " + std::to_string(attr.size());
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
for (auto &attr_val : attr) {
RETURN_IF_NOT_OK(ValidateScalar(op_name, attr_name, attr_val, range, false, false));
}
if (attr.size() == 2 && (attr[0] > attr[1])) {
std::string err_msg = op_name + ":" + attr_name +
" lower bound must be less or equal to upper bound, got lb: " + std::to_string(attr[0]) +
", ub: " + std::to_string(attr[1]);
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
} }
return Status::OK(); return Status::OK();
} }
Status ValidateProbability(const std::string &transform_name, const float &probability) { Status ValidateVectorMeanStd(const std::string &op_name, const std::vector<float> &mean,
if (probability < 0.0 || probability > 1.0) { const std::vector<float> &std) {
std::string err_msg = if (mean.size() != 3) {
transform_name + ": probability must be between 0.0 and 1.0, got: " + std::to_string(probability); std::string err_msg = op_name + ": mean expecting size 3, got size: " + std::to_string(mean.size());
MS_LOG(ERROR) << err_msg; MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg); RETURN_STATUS_SYNTAX_ERROR(err_msg);
} }
if (std.size() != 3) {
std::string err_msg = op_name + ": std expecting size 3, got size: " + std::to_string(std.size());
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
// check std/mean value
for (int32_t i = 0; i < std.size(); ++i) {
RETURN_IF_NOT_OK(ValidateScalar(op_name, "mean", mean[i], {0.0, 255.0}, false, false));
RETURN_IF_NOT_OK(ValidateScalar(op_name, "std", std[i], {0.0, 255.0}, true, false));
}
return Status::OK(); return Status::OK();
} }
Status ValidateVectorPadding(const std::string &transform_name, const std::vector<int32_t> &padding) { Status ValidateVectorPadding(const std::string &op_name, const std::vector<int32_t> &padding) {
if (padding.empty() || padding.size() == 3 || padding.size() > 4) { if (padding.empty() || padding.size() == 3 || padding.size() > 4) {
std::string err_msg = transform_name + ": padding vector has incorrect size: " + std::to_string(padding.size()); std::string err_msg = op_name + ": padding expecting size 1, 2 or 4, got size: " + std::to_string(padding.size());
MS_LOG(ERROR) << err_msg; MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg); RETURN_STATUS_SYNTAX_ERROR(err_msg);
} }
for (int32_t i = 0; i < padding.size(); ++i) { for (const auto &pad_val : padding) {
if (padding[i] < 0) { RETURN_IF_NOT_OK(ValidateScalar(op_name, "padding", pad_val, {0, INT_MAX}, false, false));
std::string err_msg =
transform_name +
": invalid padding, padding value must be greater than or equal to 0, got: " + std::to_string(padding[i]);
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
if (padding[i] == INT_MAX) {
std::string err_msg =
transform_name + ": invalid padding, padding value too large, got: " + std::to_string(padding[i]);
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
} }
return Status::OK(); return Status::OK();
} }
Status ValidateVectorPositive(const std::string &transform_name, const std::vector<int32_t> &size) { Status ValidateVectorPositive(const std::string &op_name, const std::string &vec_name,
for (int32_t i = 0; i < size.size(); ++i) { const std::vector<int32_t> &vec) {
if (size[i] <= 0) { for (const auto &vec_val : vec) {
std::string err_msg = RETURN_IF_NOT_OK(ValidateScalar(op_name, vec_name, vec_val, {0}, true));
transform_name + ": Non-positive size value: " + std::to_string(size[i]) + " at element: " + std::to_string(i); }
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg); return Status::OK();
} }
Status ValidateVectorNonNegative(const std::string &op_name, const std::string &vec_name,
const std::vector<int32_t> &vec) {
for (const auto &vec_val : vec) {
RETURN_IF_NOT_OK(ValidateScalar(op_name, vec_name, vec_val, {0}, false));
}
return Status::OK();
}
Status ValidateVectorSize(const std::string &op_name, const std::vector<int32_t> &size) {
if (size.empty() || size.size() > 2) {
std::string err_msg = op_name + ": size expecting size 2, got size.size(): " + std::to_string(size.size());
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
for (const auto &size_val : size) {
RETURN_IF_NOT_OK(ValidateScalar(op_name, "size", size_val, {0, INT_MAX}, true, false));
}
return Status::OK();
}
Status ValidateVectorScale(const std::string &op_name, const std::vector<float> &scale) {
if (scale.size() != 2) {
std::string err_msg = op_name + ": scale expecting size 2, got scale.size(): " + std::to_string(scale.size());
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
RETURN_IF_NOT_OK(ValidateScalar(op_name, "scale", scale[0], {0}, false));
RETURN_IF_NOT_OK(ValidateScalar(op_name, "scale", scale[1], {0}, true));
if (scale[1] < scale[0]) {
std::string err_msg = op_name + ": scale must be in the format of (min, max).";
MS_LOG(ERROR) << op_name + ": scale must be in the format of (min, max), but got: " << scale;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
return Status::OK();
}
Status ValidateVectorRatio(const std::string &op_name, const std::vector<float> &ratio) {
if (ratio.size() != 2) {
std::string err_msg = op_name + ": ratio expecting size 2, got ratio.size(): " + std::to_string(ratio.size());
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
RETURN_IF_NOT_OK(ValidateScalar(op_name, "scale", ratio[0], {0}, true));
RETURN_IF_NOT_OK(ValidateScalar(op_name, "scale", ratio[1], {0}, true));
if (ratio[1] < ratio[0]) {
std::string err_msg = op_name + ": ratio must be in the format of (min, max).";
MS_LOG(ERROR) << op_name + ": ratio must be in the format of (min, max), but got: " << ratio;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
} }
return Status::OK(); return Status::OK();
} }
Status ValidateVectorTransforms(const std::string &transform_name, Status ValidateVectorTransforms(const std::string &op_name,
const std::vector<std::shared_ptr<TensorOperation>> &transforms) { const std::vector<std::shared_ptr<TensorOperation>> &transforms) {
if (transforms.empty()) { if (transforms.empty()) {
std::string err_msg = transform_name + ": transform list must not be empty."; std::string err_msg = op_name + ": transform list must not be empty.";
MS_LOG(ERROR) << err_msg; MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg); RETURN_STATUS_SYNTAX_ERROR(err_msg);
} }
for (int32_t i = 0; i < transforms.size(); ++i) { for (int32_t i = 0; i < transforms.size(); ++i) {
if (transforms[i] == nullptr) { if (transforms[i] == nullptr) {
std::string err_msg = std::string err_msg =
transform_name + ": transform ops must not be null, got transform[" + std::to_string(i) + "] == nullptr."; op_name + ": transform ops must not be null, got transform[" + std::to_string(i) + "] == nullptr.";
MS_LOG(ERROR) << err_msg; MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg); RETURN_STATUS_SYNTAX_ERROR(err_msg);
} }
@ -118,7 +202,7 @@ Status ValidateVectorTransforms(const std::string &transform_name,
return Status::OK(); return Status::OK();
} }
bool CmpFloat(const float &a, const float &b, float epsilon) { return (std::fabs(a - b) < epsilon); } bool CmpFloat(const float a, const float b, float epsilon) { return (std::fabs(a - b) < epsilon); }
// Transform operations for data. // Transform operations for data.
namespace transforms { namespace transforms {

522
mindspore/ccsrc/minddata/dataset/api/vision.cc
View File

File diff suppressed because it is too large Load Diff

73
mindspore/ccsrc/minddata/dataset/include/transforms.h
Unescape View File

@ -24,6 +24,7 @@
#include "minddata/dataset/include/constants.h" #include "minddata/dataset/include/constants.h"
#include "minddata/dataset/include/status.h" #include "minddata/dataset/include/status.h"
#include "minddata/dataset/include/tensor.h"
#ifndef INCLUDE_NLOHMANN_JSON_FWD_HPP_ #ifndef INCLUDE_NLOHMANN_JSON_FWD_HPP_
#define INCLUDE_NLOHMANN_JSON_FWD_HPP_ #define INCLUDE_NLOHMANN_JSON_FWD_HPP_
@ -88,24 +89,80 @@ class TensorOperation : public std::enable_shared_from_this<TensorOperation> {
bool random_op_; bool random_op_;
}; };
// Helper function to validate probability
Status ValidateProbability(const std::string &op_name, const float probability);
// Helper function to positive int scalar
Status ValidateIntScalarPositive(const std::string &op_name, const std::string &scalar_name, int32_t scalar);
// Helper function to positive float scalar
Status ValidateFloatScalarPositive(const std::string &op_name, const std::string &scalar_name, float scalar);
// Helper function to validate scalar
template <typename T>
Status ValidateScalar(const std::string &op_name, const std::string &scalar_name, const T scalar,
const std::vector<T> &range, bool left_open_interval = false, bool right_open_interval = false) {
if (range.empty() || range.size() > 2) {
std::string err_msg = "Range check expecting size 1 or 2, but got: " + std::to_string(range.size());
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
if ((left_open_interval && scalar <= range[0]) || (!left_open_interval && scalar < range[0])) {
std::string interval_description = left_open_interval ? " greater than " : " greater than or equal to ";
std::string err_msg = op_name + ":" + scalar_name + " must be" + interval_description + std::to_string(range[0]) +
", got: " + std::to_string(scalar);
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
if (range.size() == 2) {
if ((right_open_interval && scalar >= range[1]) || (!right_open_interval && scalar > range[1])) {
std::string left_bracket = left_open_interval ? "(" : "[";
std::string right_bracket = right_open_interval ? ")" : "]";
std::string err_msg = op_name + ":" + scalar_name + " is out of range " + left_bracket +
std::to_string(range[0]) + ", " + std::to_string(range[1]) + right_bracket +
", got: " + std::to_string(scalar);
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
}
return Status::OK();
}
// Helper function to validate color attribute
Status ValidateVectorColorAttribute(const std::string &op_name, const std::string &attr_name,
const std::vector<float> &attr, const std::vector<float> &range);
// Helper function to validate fill value // Helper function to validate fill value
Status ValidateVectorFillvalue(const std::string &transform_name, const std::vector<uint8_t> &fill_value); Status ValidateVectorFillvalue(const std::string &op_name, const std::vector<uint8_t> &fill_value);
// Helper function to validate probability // Helper function to validate mean/std value
Status ValidateProbability(const std::string &transform_name, const float &probability); Status ValidateVectorMeanStd(const std::string &op_name, const std::vector<float> &mean, const std::vector<float> &std);
// Helper function to validate padding // Helper function to validate padding
Status ValidateVectorPadding(const std::string &transform_name, const std::vector<int32_t> &padding); Status ValidateVectorPadding(const std::string &op_name, const std::vector<int32_t> &padding);
// Helper function to validate positive value
Status ValidateVectorPositive(const std::string &op_name, const std::string &vec_name, const std::vector<int32_t> &vec);
// Helper function to validate non-negative value
Status ValidateVectorNonNegative(const std::string &op_name, const std::string &vec_name,
const std::vector<int32_t> &vec);
// Helper function to validate size of size
Status ValidateVectorSize(const std::string &op_name, const std::vector<int32_t> &size);
// Helper function to validate scale
Status ValidateVectorScale(const std::string &op_name, const std::vector<float> &scale);
// Helper function to validate size // Helper function to validate ratio
Status ValidateVectorPositive(const std::string &transform_name, const std::vector<int32_t> &size); Status ValidateVectorRatio(const std::string &op_name, const std::vector<float> &ratio);
// Helper function to validate transforms // Helper function to validate transforms
Status ValidateVectorTransforms(const std::string &transform_name, Status ValidateVectorTransforms(const std::string &op_name,
const std::vector<std::shared_ptr<TensorOperation>> &transforms); const std::vector<std::shared_ptr<TensorOperation>> &transforms);
// Helper function to compare float value // Helper function to compare float value
bool CmpFloat(const float &a, const float &b, float epsilon = 0.0000000001f); bool CmpFloat(const float a, const float b, float epsilon = 0.0000000001f);
// Transform operations for performing data transformation. // Transform operations for performing data transformation.
namespace transforms { namespace transforms {

51
tests/ut/cpp/dataset/c_api_vision_test.cc
Unescape View File

@ -265,6 +265,9 @@ TEST_F(MindDataTestPipeline, TestCropFail) {
// zero height // zero height
crop = mindspore::dataset::vision::Crop({0, 0}, {0, 32}); crop = mindspore::dataset::vision::Crop({0, 0}, {0, 32});
EXPECT_EQ(crop, nullptr); EXPECT_EQ(crop, nullptr);
// negative coordinates
crop = mindspore::dataset::vision::Crop({-1, 0}, {32, 32});
EXPECT_EQ(crop, nullptr);
} }
TEST_F(MindDataTestPipeline, TestCutMixBatchSuccess1) { TEST_F(MindDataTestPipeline, TestCutMixBatchSuccess1) {
@ -882,7 +885,7 @@ TEST_F(MindDataTestPipeline, TestNormalize) {
EXPECT_NE(ds, nullptr); EXPECT_NE(ds, nullptr);
// Create objects for the tensor ops // Create objects for the tensor ops
std::shared_ptr<TensorOperation> normalize = vision::Normalize({121.0, 115.0, 100.0}, {70.0, 68.0, 71.0}); std::shared_ptr<TensorOperation> normalize = vision::Normalize({121.0, 115.0, 0.0}, {70.0, 68.0, 71.0});
EXPECT_NE(normalize, nullptr); EXPECT_NE(normalize, nullptr);
// Create a Map operation on ds // Create a Map operation on ds
@ -924,6 +927,15 @@ TEST_F(MindDataTestPipeline, TestNormalizeFail) {
std::shared_ptr<TensorOperation> normalize = std::shared_ptr<TensorOperation> normalize =
mindspore::dataset::vision::Normalize({121.0, 115.0, 100.0}, {0.0, 68.0, 71.0}); mindspore::dataset::vision::Normalize({121.0, 115.0, 100.0}, {0.0, 68.0, 71.0});
EXPECT_EQ(normalize, nullptr); EXPECT_EQ(normalize, nullptr);
// mean out of range
normalize = mindspore::dataset::vision::Normalize({121.0, 0.0, 100.0}, {256.0, 68.0, 71.0});
EXPECT_EQ(normalize, nullptr);
// mean out of range
normalize = mindspore::dataset::vision::Normalize({256.0, 0.0, 100.0}, {70.0, 68.0, 71.0});
EXPECT_EQ(normalize, nullptr);
// mean out of range
normalize = mindspore::dataset::vision::Normalize({-1.0, 0.0, 100.0}, {70.0, 68.0, 71.0});
EXPECT_EQ(normalize, nullptr);
// normalize with 2 values (not 3 values) for mean // normalize with 2 values (not 3 values) for mean
normalize = mindspore::dataset::vision::Normalize({121.0, 115.0}, {70.0, 68.0, 71.0}); normalize = mindspore::dataset::vision::Normalize({121.0, 115.0}, {70.0, 68.0, 71.0});
EXPECT_EQ(normalize, nullptr); EXPECT_EQ(normalize, nullptr);
@ -1258,7 +1270,7 @@ TEST_F(MindDataTestPipeline, TestRandomColorAdjust) {
std::shared_ptr<TensorOperation> random_color_adjust4 = vision::RandomColorAdjust(); std::shared_ptr<TensorOperation> random_color_adjust4 = vision::RandomColorAdjust();
EXPECT_NE(random_color_adjust4, nullptr); EXPECT_NE(random_color_adjust4, nullptr);
// Use subset of explictly set parameters // Use subset of explicitly set parameters
std::shared_ptr<TensorOperation> random_color_adjust5 = vision::RandomColorAdjust({0.0, 0.5}, {0.25}); std::shared_ptr<TensorOperation> random_color_adjust5 = vision::RandomColorAdjust({0.0, 0.5}, {0.25});
EXPECT_NE(random_color_adjust5, nullptr); EXPECT_NE(random_color_adjust5, nullptr);
@ -1295,6 +1307,31 @@ TEST_F(MindDataTestPipeline, TestRandomColorAdjust) {
iter->Stop(); iter->Stop();
} }
TEST_F(MindDataTestPipeline, TestRandomColorAdjustFail) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomColorAdjustFail.";
// brightness out of range
std::shared_ptr<TensorOperation> random_color_adjust1 = vision::RandomColorAdjust({-1.0});
EXPECT_EQ(random_color_adjust1, nullptr);
// contrast out of range
std::shared_ptr<TensorOperation> random_color_adjust2 = vision::RandomColorAdjust({1.0}, {-0.1});
EXPECT_EQ(random_color_adjust2, nullptr);
// saturation out of range
std::shared_ptr<TensorOperation> random_color_adjust3 = vision::RandomColorAdjust({0.0}, {0.0}, {-0.2});
EXPECT_EQ(random_color_adjust3, nullptr);
// hue out of range
std::shared_ptr<TensorOperation> random_color_adjust4 = vision::RandomColorAdjust({0.0}, {0.0}, {0.0}, {-0.6});
EXPECT_EQ(random_color_adjust4, nullptr);
std::shared_ptr<TensorOperation> random_color_adjust5 = vision::RandomColorAdjust({0.0}, {0.0}, {0.0}, {-0.5, 0.6});
EXPECT_EQ(random_color_adjust5, nullptr);
std::shared_ptr<TensorOperation> random_color_adjust6 = vision::RandomColorAdjust({0.0}, {0.0}, {0.0}, {0.51});
EXPECT_EQ(random_color_adjust6, nullptr);
}
TEST_F(MindDataTestPipeline, TestRandomCropSuccess) { TEST_F(MindDataTestPipeline, TestRandomCropSuccess) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomCropSuccess."; MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRandomCropSuccess.";
// Create an VOC Dataset // Create an VOC Dataset
@ -1796,13 +1833,17 @@ TEST_F(MindDataTestPipeline, TestRandomResizeFail) {
std::shared_ptr<TensorOperation> random_resize1 = vision::RandomResize({-66, 77}); std::shared_ptr<TensorOperation> random_resize1 = vision::RandomResize({-66, 77});
EXPECT_EQ(random_resize1, nullptr); EXPECT_EQ(random_resize1, nullptr);
// RandomResize : size must be a vector of one or two values // RandomResize : size must only contain positive integers
std::shared_ptr<TensorOperation> random_resize2 = vision::RandomResize({1, 2, 3}); std::shared_ptr<TensorOperation> random_resize2 = vision::RandomResize({0, 77});
EXPECT_EQ(random_resize2, nullptr); EXPECT_EQ(random_resize2, nullptr);
// RandomResize : size must be a vector of one or two values // RandomResize : size must be a vector of one or two values
std::shared_ptr<TensorOperation> random_resize3 = vision::RandomResize({}); std::shared_ptr<TensorOperation> random_resize3 = vision::RandomResize({1, 2, 3});
EXPECT_EQ(random_resize3, nullptr); EXPECT_EQ(random_resize3, nullptr);
// RandomResize : size must be a vector of one or two values
std::shared_ptr<TensorOperation> random_resize4 = vision::RandomResize({});
EXPECT_EQ(random_resize4, nullptr);
} }
TEST_F(MindDataTestPipeline, TestRandomResizeWithBBoxSuccess1) { TEST_F(MindDataTestPipeline, TestRandomResizeWithBBoxSuccess1) {

Write Preview
Loading…
Cancel
Save