Add error message

mindspore/ccsrc/minddata/dataset/api/execute.cc

@@ -297,11 +297,6 @@ Status Execute::operator()(const mindspore::MSTensor &input, mindspore::MSTensor
     }
     CHECK_FAIL_RETURN_UNEXPECTED(device_input->HasDeviceData(), "Apply transform failed, output tensor has no data");
 
-    // TODO(lizhenglong) waiting for computing department development, hence we pop data onto host temporarily.
-    // std::shared_ptr<mindspore::dataset::Tensor> host_output;
-    // RETURN_IF_NOT_OK(device_resource_->Pop(device_input, &host_output));
-    // *output = mindspore::MSTensor(std::make_shared<DETensor>(host_output));
-
     *output = mindspore::MSTensor(std::make_shared<DETensor>(device_input, true));
 #endif
   }

mindspore/ccsrc/minddata/dataset/core/ascend_resource.cc

@@ -61,7 +61,9 @@ Status AscendResource::Sink(const mindspore::MSTensor &host_input, std::shared_p
                                                 (const uchar *)(host_input.Data().get()), &de_input);
   RETURN_IF_NOT_OK(rc);
   if (!IsNonEmptyJPEG(de_input)) {
-    RETURN_STATUS_UNEXPECTED("Dvpp operators can only support processing JPEG image");
+    std::string err_msg = "Dvpp operators can only support processing JPEG image";
+    MS_LOG(ERROR) << err_msg;
+    RETURN_STATUS_UNEXPECTED(err_msg);
   }
 
   APP_ERROR ret = processor_->H2D_Sink(de_input, *device_input);

mindspore/ccsrc/minddata/dataset/core/de_tensor.cc

@@ -23,10 +23,10 @@
 #include "utils/hashing.h"
 #ifndef ENABLE_ANDROID
 #include "utils/log_adapter.h"
-#define ASSERT_NULL(ptr) MS_EXCEPTION_IF_NULL(ptr)
+#define EXCEPTION_IF_NULL(ptr) MS_EXCEPTION_IF_NULL(ptr)
 #else
 #include "mindspore/lite/src/common/log_adapter.h"
-#define ASSERT_NULL(ptr) MS_ASSERT((ptr) != nullptr)
+#define EXCEPTION_IF_NULL(ptr) MS_ASSERT((ptr) != nullptr)
 #endif
 
 namespace mindspore {
@@ -63,22 +63,22 @@ const std::string &DETensor::Name() const { return name_; }
 enum mindspore::DataType DETensor::DataType() const {
 #ifndef ENABLE_ANDROID
   if (is_device_) {
-    ASSERT_NULL(device_tensor_impl_);
+    EXCEPTION_IF_NULL(device_tensor_impl_);
     return static_cast<mindspore::DataType>(DETypeToMSType(device_tensor_impl_->DeviceDataType()));
   }
 #endif
-  ASSERT_NULL(tensor_impl_);
+  EXCEPTION_IF_NULL(tensor_impl_);
   return static_cast<mindspore::DataType>(DETypeToMSType(tensor_impl_->type()));
 }
 
 size_t DETensor::DataSize() const {
 #ifndef ENABLE_ANDROID
   if (is_device_) {
-    ASSERT_NULL(device_tensor_impl_);
+    EXCEPTION_IF_NULL(device_tensor_impl_);
     return device_tensor_impl_->DeviceDataSize();
   }
 #endif
-  ASSERT_NULL(tensor_impl_);
+  EXCEPTION_IF_NULL(tensor_impl_);
   return tensor_impl_->SizeInBytes();
 }
 
@@ -87,7 +87,7 @@ const std::vector<int64_t> &DETensor::Shape() const { return shape_; }
 std::shared_ptr<const void> DETensor::Data() const {
 #ifndef ENABLE_ANDROID
   if (is_device_) {
-    ASSERT_NULL(device_tensor_impl_);
+    EXCEPTION_IF_NULL(device_tensor_impl_);
     return std::shared_ptr<const void>(device_tensor_impl_->GetHostBuffer(), [](const void *) {});
   }
 #endif
@@ -97,11 +97,11 @@ std::shared_ptr<const void> DETensor::Data() const {
 void *DETensor::MutableData() {
 #ifndef ENABLE_ANDROID
   if (is_device_) {
-    ASSERT_NULL(device_tensor_impl_);
+    EXCEPTION_IF_NULL(device_tensor_impl_);
     return static_cast<void *>(device_tensor_impl_->GetDeviceMutableBuffer());
   }
 #endif
-  ASSERT_NULL(tensor_impl_);
+  EXCEPTION_IF_NULL(tensor_impl_);
   return static_cast<void *>(tensor_impl_->GetMutableBuffer());
 }
 
@@ -110,7 +110,7 @@ bool DETensor::IsDevice() const { return is_device_; }
 std::shared_ptr<mindspore::MSTensor::Impl> DETensor::Clone() const {
 #ifndef ENABLE_ANDROID
   if (is_device_) {
-    ASSERT_NULL(device_tensor_impl_);
+    EXCEPTION_IF_NULL(device_tensor_impl_);
     return std::make_shared<DETensor>(device_tensor_impl_, is_device_);
   }
 #endif

mindspore/ccsrc/minddata/dataset/core/device_tensor.cc

@@ -98,7 +98,7 @@ const unsigned char *DeviceTensor::GetHostBuffer() {
   return host_data_tensor_->GetBuffer();
 }
 
-uint8_t *DeviceTensor::GetDeviceBuffer() { return device_data_; }
+const uint8_t *DeviceTensor::GetDeviceBuffer() { return device_data_; }
 
 uint8_t *DeviceTensor::GetDeviceMutableBuffer() { return device_data_; }
 
@@ -136,6 +136,7 @@ Status DeviceTensor::DataPop_(std::shared_ptr<Tensor> *host_tensor) {
     MS_LOG(ERROR) << "Failed to allocate memory from host ret = " << ret;
     return Status(StatusCode::kMDNoSpace);
   }
+
   std::shared_ptr<void> outBuf(resHostBuf, aclrtFreeHost);
   auto processedInfo_ = outBuf;
   // Memcpy the output data from device to host
@@ -145,6 +146,7 @@ Status DeviceTensor::DataPop_(std::shared_ptr<Tensor> *host_tensor) {
     MS_LOG(ERROR) << "Failed to copy memory from device to host, ret = " << ret;
     return Status(StatusCode::kMDOutOfMemory);
   }
+
   auto data = std::static_pointer_cast<unsigned char>(processedInfo_);
   unsigned char *ret_ptr = data.get();
 
@@ -155,11 +157,14 @@ Status DeviceTensor::DataPop_(std::shared_ptr<Tensor> *host_tensor) {
   uint32_t _output_height_ = this->GetYuvStrideShape()[2];
   uint32_t _output_heightStride_ = this->GetYuvStrideShape()[3];
   const mindspore::dataset::DataType dvpp_data_type(mindspore::dataset::DataType::DE_UINT8);
+
   mindspore::dataset::Tensor::CreateFromMemory(dvpp_shape, dvpp_data_type, ret_ptr, host_tensor);
+
   (*host_tensor)->SetYuvShape(_output_width_, _output_widthStride_, _output_height_, _output_heightStride_);
   if (!(*host_tensor)->HasData()) {
     return Status(StatusCode::kMCDeviceError);
   }
+
   MS_LOG(INFO) << "Successfully pop DeviceTensor data onto host";
   return Status::OK();
 }

mindspore/ccsrc/minddata/dataset/core/device_tensor.h

@@ -45,7 +45,7 @@ class DeviceTensor : public Tensor {
 
   const unsigned char *GetHostBuffer();
 
-  uint8_t *GetDeviceBuffer();
+  const uint8_t *GetDeviceBuffer();
 
   uint8_t *GetDeviceMutableBuffer();
 

mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/dvpp_crop_jpeg_op.cc

@@ -61,6 +61,9 @@ Status DvppCropJpegOp::Compute(const std::shared_ptr<DeviceTensor> &input, std::
 
 Status DvppCropJpegOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
   IO_CHECK(input, output);
+  if (!IsNonEmptyJPEG(input)) {
+    RETURN_STATUS_UNEXPECTED("DvppCropJpegOp only support process jpeg image.");
+  }
   try {
     CHECK_FAIL_RETURN_UNEXPECTED(input->GetBuffer() != nullptr, "The input image buffer is empty.");
     unsigned char *buffer = const_cast<unsigned char *>(input->GetBuffer());

mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/dvpp_decode_jpeg_op.cc

@@ -55,6 +55,7 @@ Status DvppDecodeJpegOp::Compute(const std::shared_ptr<DeviceTensor> &input, std
   }
   return Status::OK();
 }
+
 // Compute() will be called when context=="CPU"
 Status DvppDecodeJpegOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
   IO_CHECK(input, output);

mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/dvpp_decode_resize_crop_jpeg_op.cc

@@ -59,7 +59,7 @@ Status DvppDecodeResizeCropJpegOp::Compute(const std::shared_ptr<DeviceTensor> &
 Status DvppDecodeResizeCropJpegOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
   IO_CHECK(input, output);
   if (!IsNonEmptyJPEG(input)) {
-    RETURN_STATUS_UNEXPECTED("DvppDecodeReiszeJpegOp only support process jpeg image.");
+    RETURN_STATUS_UNEXPECTED("DvppDecodeReiszeCropJpegOp only support process jpeg image.");
   }
   try {
     CHECK_FAIL_RETURN_UNEXPECTED(input->GetBuffer() != nullptr, "The input image buffer is empty.");

mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/dvpp_normalize_op.cc

@@ -24,8 +24,8 @@ Status DvppNormalizeOp::Compute(const std::shared_ptr<DeviceTensor> &input, std:
   const DataType dvpp_data_type(DataType::DE_UINT8);
   mindspore::dataset::DeviceTensor::CreateEmpty(dvpp_shape, dvpp_data_type, output);
   std::vector<uint32_t> yuv_shape = input->GetYuvStrideShape();
-  (*output)->SetAttributes(input->GetDeviceBuffer(), input->DeviceDataSize(), yuv_shape[0], yuv_shape[1], yuv_shape[2],
+  (*output)->SetAttributes(input->GetDeviceMutableBuffer(), input->DeviceDataSize(), yuv_shape[0], yuv_shape[1],
-                           yuv_shape[3]);
+                           yuv_shape[2], yuv_shape[3]);
   if (!((*output)->HasDeviceData())) {
     std::string error = "[ERROR] Fail to get the output result from device memory!";
     RETURN_STATUS_UNEXPECTED(error);

mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/dvpp_resize_jpeg_op.cc

@@ -62,6 +62,9 @@ Status DvppResizeJpegOp::Compute(const std::shared_ptr<DeviceTensor> &input, std
 
 Status DvppResizeJpegOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
   IO_CHECK(input, output);
+  if (!IsNonEmptyJPEG(input)) {
+    RETURN_STATUS_UNEXPECTED("DvppReiszeJpegOp only support process jpeg image.");
+  }
   try {
     CHECK_FAIL_RETURN_UNEXPECTED(input->GetBuffer() != nullptr, "The input image buffer is empty.");
     unsigned char *buffer = const_cast<unsigned char *>(input->GetBuffer());

tests/st/cpp/dataset/test_de.cc

@@ -13,6 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+#include <iostream>
 #include <string>
 #include <vector>
 #include "common/common_test.h"
@@ -37,11 +38,35 @@ class TestDE : public ST::Common {
   TestDE() {}
 };
 
+mindspore::MSTensor ReadFileToTensor(const std::string &file) {
+  if (file.empty()) {
+    std::cout << "[ERROR]Pointer file is nullptr, return an empty Tensor." << std::endl;
+    return mindspore::MSTensor();
+  }
+  std::ifstream ifs(file);
+  if (!ifs.good()) {
+    std::cout << "[ERROR]File: " << file << " does not exist, return an empty Tensor." << std::endl;
+    return mindspore::MSTensor();
+  }
+  if (!ifs.is_open()) {
+    std::cout << "[ERROR]File: " << file << "open failed, return an empty Tensor." << std::endl;
+    return mindspore::MSTensor();
+  }
+
+  ifs.seekg(0, std::ios::end);
+  size_t size = ifs.tellg();
+  mindspore::MSTensor buf("file", mindspore::DataType::kNumberTypeUInt8, {static_cast<int64_t>(size)}, nullptr, size);
+
+  ifs.seekg(0, std::ios::beg);
+  ifs.read(reinterpret_cast<char *>(buf.MutableData()), size);
+  ifs.close();
+
+  return buf;
+}
+
 TEST_F(TestDE, TestResNetPreprocess) {
   // Read images
-  std::shared_ptr<mindspore::dataset::Tensor> de_tensor;
+  auto image = ReadFileToTensor("./data/dataset/apple.jpg");
-  mindspore::dataset::Tensor::CreateFromFile("./data/dataset/apple.jpg", &de_tensor);
-  auto image = mindspore::MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_tensor));
 
   // Define transform operations
   std::shared_ptr<TensorTransform> decode(new vision::Decode());
@@ -66,10 +91,15 @@ TEST_F(TestDE, TestResNetPreprocess) {
 TEST_F(TestDE, TestDvpp) {
 #ifdef ENABLE_ACL
   // Read images from target directory
+
+  /* Old internal method, we deprecate it
   std::shared_ptr<mindspore::dataset::Tensor> de_tensor;
   Status rc = mindspore::dataset::Tensor::CreateFromFile("./data/dataset/apple.jpg", &de_tensor);
   ASSERT_TRUE(rc.IsOk());
   auto image = MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_tensor));
+  */
+
+  auto image = ReadFileToTensor("./data/dataset/apple.jpg");
 
   // Define dvpp transform
   std::vector<uint32_t> crop_paras = {224, 224};
@@ -78,7 +108,7 @@ TEST_F(TestDE, TestDvpp) {
   mindspore::dataset::Execute Transform(decode_resize_crop, MapTargetDevice::kAscend310);
 
   // Apply transform on images
-  rc = Transform(image, &image);
+  Status rc = Transform(image, &image);
   std::string aipp_cfg = Transform.AippCfgGenerator();
   ASSERT_EQ(aipp_cfg, "./aipp.cfg");
 
@@ -116,10 +146,7 @@ TEST_F(TestDE, TestDvpp) {
 TEST_F(TestDE, TestDvppSinkMode) {
 #ifdef ENABLE_ACL
   // Read images from target directory
-  std::shared_ptr<mindspore::dataset::Tensor> de_tensor;
+  auto image = ReadFileToTensor("./data/dataset/apple.jpg");
-  Status rc = mindspore::dataset::Tensor::CreateFromFile("./data/dataset/apple.jpg", &de_tensor);
-  ASSERT_TRUE(rc.IsOk());
-  auto image = MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_tensor));
 
   // Define dvpp transform
   std::vector<int32_t> crop_paras = {224, 224};
@@ -131,7 +158,7 @@ TEST_F(TestDE, TestDvppSinkMode) {
   mindspore::dataset::Execute Transform(trans_list, MapTargetDevice::kAscend310);
 
   // Apply transform on images
-  rc = Transform(image, &image);
+  Status rc = Transform(image, &image);
 
   // Check image info
   ASSERT_TRUE(rc.IsOk());
@@ -159,10 +186,7 @@ TEST_F(TestDE, TestDvppSinkMode) {
 
 TEST_F(TestDE, TestDvppDecodeResizeCropNormalize) {
 #ifdef ENABLE_ACL
-  std::shared_ptr<mindspore::dataset::Tensor> de_tensor;
+  auto image = ReadFileToTensor("./data/dataset/apple.jpg");
-  Status rc = mindspore::dataset::Tensor::CreateFromFile("./data/dataset/apple.jpg", &de_tensor);
-  ASSERT_TRUE(rc.IsOk());
-  auto image = MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_tensor));
 
   // Define dvpp transform
   std::vector<int32_t> crop_paras = {416};
@@ -182,7 +206,7 @@ TEST_F(TestDE, TestDvppDecodeResizeCropNormalize) {
   ASSERT_EQ(aipp_cfg, "./aipp.cfg");
 
   // Apply transform on images
-  rc = Transform(image, &image);
+  Status rc = Transform(image, &image);
 
   // Check image info
   ASSERT_TRUE(rc.IsOk());