m53297601
/
mindspore
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

fix minddata include internal head file & fix execute ut

Browse Source
pull/12760/head
YangLuo 4 years ago
parent 00f25c8409
commit eecbc99b2e
6 changed files with 66 additions and 43 deletions
Show Stats Download Patch File Download Diff File

6
mindspore/ccsrc/minddata/dataset/api/execute.cc
Unescape View File

@ -83,10 +83,9 @@ Execute::Execute(std::shared_ptr<TensorTransform> op, MapTargetDevice deviceType
#endif
}
/*
Execute::Execute(TensorTransform op, MapTargetDevice deviceType) {
Execute::Execute(std::reference_wrapper<TensorTransform> op, MapTargetDevice deviceType) {
// Convert op from TensorTransform to TensorOperation
std::shared_ptr<TensorOperation> operation = op.Parse();
std::shared_ptr<TensorOperation> operation = op.get().Parse();
ops_.emplace_back(std::move(operation));
device_type_ = deviceType;
#ifdef ENABLE_ACL
@ -100,7 +99,6 @@ Execute::Execute(TensorTransform op, MapTargetDevice deviceType) {
}
#endif
}
*/
// Execute function for the example case: auto decode(new vision::Decode());
Execute::Execute(TensorTransform *op, MapTargetDevice deviceType) {

2
mindspore/ccsrc/minddata/dataset/engine/ir/cache/dataset_cache.h vendored
Unescape View File

@ -19,7 +19,7 @@
#include <memory>
#include "minddata/dataset/engine/datasetops/dataset_op.h"
#include "minddata/dataset/include/samplers.h"
#include "minddata/dataset/engine/ir/datasetops/source/samplers/samplers_ir.h"
#include "minddata/dataset/util/status.h"
namespace mindspore::dataset {

2
mindspore/ccsrc/minddata/dataset/include/execute.h
Unescape View File

@ -36,7 +36,7 @@ class Execute {
// FIXME - Temporarily overload Execute to support both TensorOperation and TensorTransform
explicit Execute(std::shared_ptr<TensorOperation> op, MapTargetDevice deviceType = MapTargetDevice::kCpu);
explicit Execute(std::shared_ptr<TensorTransform> op, MapTargetDevice deviceType = MapTargetDevice::kCpu);
// explicit Execute(TensorTransform op, MapTargetDevice deviceType = MapTargetDevice::KCpu);
explicit Execute(std::reference_wrapper<TensorTransform> op, MapTargetDevice deviceType = MapTargetDevice::kCpu);
explicit Execute(TensorTransform *op, MapTargetDevice deviceType = MapTargetDevice::kCpu);
explicit Execute(std::vector<std::shared_ptr<TensorOperation>> ops,

7
mindspore/ccsrc/minddata/dataset/include/samplers.h
Unescape View File

@ -20,11 +20,12 @@
#include <memory>
#include <vector>
// FIXME - This internal IR header will be removed when external API classes are provided
#include "minddata/dataset/engine/ir/datasetops/source/samplers/samplers_ir.h"
namespace mindspore {
namespace dataset {
// Forward declare
class SamplerObj;
// Abstract class to represent a sampler in the data pipeline.
/// \class Sampler samplers.h
/// \brief An abstract base class to represent a sampler in the data pipeline.

1
tests/ut/cpp/dataset/c_api_samplers_test.cc
Unescape View File

@ -15,6 +15,7 @@
*/
#include "common/common.h"
#include "minddata/dataset/engine/datasetops/source/sampler/sampler.h"
#include "minddata/dataset/engine/ir/datasetops/source/samplers/samplers_ir.h"
#include "minddata/dataset/include/datasets.h"
#include <functional>

91
tests/ut/cpp/dataset/execute_test.cc
Unescape View File

@ -13,8 +13,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <fstream>
#include "common/common.h"
#include "common/cvop_common.h"
#include "include/api/types.h"
#include "minddata/dataset/core/de_tensor.h"
#include "minddata/dataset/include/execute.h"
#include "minddata/dataset/include/transforms.h"
@ -33,12 +35,37 @@ class MindDataTestExecute : public UT::CVOP::CVOpCommon {
std::shared_ptr<Tensor> output_tensor_;
};
mindspore::MSTensor ReadFileToTensor(const std::string &file) {
if (file.empty()) {
MS_LOG(ERROR) << "Pointer file is nullptr, return an empty Tensor.";
return mindspore::MSTensor();
}
std::ifstream ifs(file);
if (!ifs.good()) {
MS_LOG(ERROR) << "File: " << file << " does not exist, return an empty Tensor.";
return mindspore::MSTensor();
}
if (!ifs.is_open()) {
MS_LOG(ERROR) << "File: " << file << "open failed, return an empty Tensor.";
return mindspore::MSTensor();
}
ifs.seekg(0, std::ios::end);
size_t size = ifs.tellg();
mindspore::MSTensor buf("file", mindspore::DataType::kNumberTypeUInt8, {static_cast<int64_t>(size)}, nullptr, size);
ifs.seekg(0, std::ios::beg);
ifs.read(reinterpret_cast<char *>(buf.MutableData()), size);
ifs.close();
return buf;
}
TEST_F(MindDataTestExecute, TestComposeTransforms) {
MS_LOG(INFO) << "Doing TestComposeTransforms.";
std::shared_ptr<mindspore::dataset::Tensor> de_tensor;
mindspore::dataset::Tensor::CreateFromFile("data/dataset/apple.jpg", &de_tensor);
auto image = mindspore::MSTensor(std::make_shared<DETensor>(de_tensor));
// Read images
auto image = ReadFileToTensor("data/dataset/apple.jpg");
// Transform params
std::shared_ptr<TensorTransform> decode = std::make_shared<vision::Decode>();
@ -59,9 +86,7 @@ TEST_F(MindDataTestExecute, TestTransformInput1) {
// instantiated via mix of make_shared and new
// Read images
std::shared_ptr<mindspore::dataset::Tensor> de_tensor;
mindspore::dataset::Tensor::CreateFromFile("data/dataset/apple.jpg", &de_tensor);
auto image = mindspore::MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_tensor));
auto image = ReadFileToTensor("data/dataset/apple.jpg");
// Define transform operations
std::shared_ptr<TensorTransform> decode = std::make_shared<vision::Decode>();
@ -88,7 +113,7 @@ TEST_F(MindDataTestExecute, TestTransformInput2) {
// Test Execute with transform op input using API constructors, with std::shared_ptr<TensorTransform pointers,
// instantiated via new
// Read images
// Read image, construct MSTensor from dataset tensor
std::shared_ptr<mindspore::dataset::Tensor> de_tensor;
mindspore::dataset::Tensor::CreateFromFile("data/dataset/apple.jpg", &de_tensor);
auto image = mindspore::MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_tensor));
@ -117,30 +142,30 @@ TEST_F(MindDataTestExecute, TestTransformInput3) {
MS_LOG(INFO) << "Doing MindDataTestExecute-TestTransformInput3.";
// Test Execute with transform op input using API constructors, with auto pointers
// Read image
// Read image, construct MSTensor from dataset tensor
std::shared_ptr<mindspore::dataset::Tensor> de_tensor;
mindspore::dataset::Tensor::CreateFromFile("data/dataset/apple.jpg", &de_tensor);
auto image = mindspore::MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_tensor));
// Define transform operations
auto decode(new vision::Decode()); // auto will create raw pointer to Decode class
auto resize(new vision::Resize({224, 224}));
auto normalize(
new vision::Normalize({0.485 * 255, 0.456 * 255, 0.406 * 255}, {0.229 * 255, 0.224 * 255, 0.225 * 255}));
auto hwc2chw(new vision::HWC2CHW());
auto decode = vision::Decode();
mindspore::dataset::Execute Transform1(decode);
std::vector<TensorTransform *> op_list = {decode, resize, normalize, hwc2chw};
mindspore::dataset::Execute Transform(op_list);
auto resize = vision::Resize({224, 224});
mindspore::dataset::Execute Transform2(resize);
// Apply transform on image
Status rc = Transform(image, &image);
Status rc;
rc = Transform1(image, &image);
ASSERT_TRUE(rc.IsOk());
rc = Transform2(image, &image);
ASSERT_TRUE(rc.IsOk());
// Check image info
ASSERT_TRUE(rc.IsOk());
ASSERT_EQ(image.Shape().size(), 3);
ASSERT_EQ(image.Shape()[0], 3);
ASSERT_EQ(image.Shape()[0], 224);
ASSERT_EQ(image.Shape()[1], 224);
ASSERT_EQ(image.Shape()[2], 224);
ASSERT_EQ(image.Shape()[2], 3);
}
TEST_F(MindDataTestExecute, TestTransformInputSequential) {
@ -148,28 +173,26 @@ TEST_F(MindDataTestExecute, TestTransformInputSequential) {
// Test Execute with transform op input using API constructors, with auto pointers;
// Apply 2 transformations sequentially, including single non-vector Transform op input
// Read images
// Read image, construct MSTensor from dataset tensor
std::shared_ptr<mindspore::dataset::Tensor> de_tensor;
mindspore::dataset::Tensor::CreateFromFile("data/dataset/apple.jpg", &de_tensor);
auto image = mindspore::MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_tensor));
// Define transform#1 operations
auto decode(new vision::Decode()); // auto will create raw pointer to Decode class
auto resize(new vision::Resize({224, 224}));
auto normalize(
std::shared_ptr<TensorTransform> decode(new vision::Decode());
std::shared_ptr<TensorTransform> resize(new vision::Resize({224, 224}));
std::shared_ptr<TensorTransform> normalize(
new vision::Normalize({0.485 * 255, 0.456 * 255, 0.406 * 255}, {0.229 * 255, 0.224 * 255, 0.225 * 255}));
std::vector<TensorTransform *> op_list = {decode, resize, normalize};
std::vector<std::shared_ptr<TensorTransform>> op_list = {decode, resize, normalize};
mindspore::dataset::Execute Transform(op_list);
// Apply transform#1 on image
Status rc = Transform(image, &image);
// Define transform#2 operations
auto hwc2chw(new vision::HWC2CHW());
TensorTransform *op_single = hwc2chw;
mindspore::dataset::Execute Transform2(op_single);
std::shared_ptr<TensorTransform> hwc2chw(new vision::HWC2CHW());
mindspore::dataset::Execute Transform2(hwc2chw);
// Apply transform#2 on image
rc = Transform2(image, &image);
@ -186,7 +209,7 @@ TEST_F(MindDataTestExecute, TestTransformDecodeResizeCenterCrop1) {
MS_LOG(INFO) << "Doing MindDataTestExecute-TestTransformDecodeResizeCenterCrop1.";
// Test Execute with Decode, Resize and CenterCrop transform ops input using API constructors, with auto pointers
// Read image
// Read image, construct MSTensor from dataset tensor
std::shared_ptr<mindspore::dataset::Tensor> de_tensor;
mindspore::dataset::Tensor::CreateFromFile("data/dataset/apple.jpg", &de_tensor);
auto image = mindspore::MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_tensor));
@ -194,12 +217,12 @@ TEST_F(MindDataTestExecute, TestTransformDecodeResizeCenterCrop1) {
// Define transform operations
std::vector<int32_t> resize_paras = {256, 256};
std::vector<int32_t> crop_paras = {224, 224};
auto decode(new vision::Decode());
auto resize(new vision::Resize(resize_paras));
auto centercrop(new vision::CenterCrop(crop_paras));
auto hwc2chw(new vision::HWC2CHW());
std::shared_ptr<TensorTransform> decode(new vision::Decode());
std::shared_ptr<TensorTransform> resize(new vision::Resize(resize_paras));
std::shared_ptr<TensorTransform> centercrop(new vision::CenterCrop(crop_paras));
std::shared_ptr<TensorTransform> hwc2chw(new vision::HWC2CHW());
std::vector<TensorTransform *> op_list = {decode, resize, centercrop, hwc2chw};
std::vector<std::shared_ptr<TensorTransform>> op_list = {decode, resize, centercrop, hwc2chw};
mindspore::dataset::Execute Transform(op_list, MapTargetDevice::kCpu);
// Apply transform on image

Write Preview
Loading…
Cancel
Save