From 7bc35b01f2d87cb1d3f458499dc967fde62258ba Mon Sep 17 00:00:00 2001 From: xiefangqi Date: Tue, 2 Feb 2021 21:51:18 +0800 Subject: [PATCH] fix error log issue --- .../ccsrc/minddata/dataset/core/data_type.cc | 4 +- .../ccsrc/minddata/dataset/core/tensor.cc | 75 +++++++++++-------- .../ccsrc/minddata/dataset/core/tensor.h | 7 ++ .../ccsrc/minddata/dataset/include/tensor.h | 54 ++++++++++--- .../dataset/vision/py_transforms_util.py | 5 ++ 5 files changed, 102 insertions(+), 43 deletions(-) diff --git a/mindspore/ccsrc/minddata/dataset/core/data_type.cc b/mindspore/ccsrc/minddata/dataset/core/data_type.cc index 9608caec25..80041253f0 100644 --- a/mindspore/ccsrc/minddata/dataset/core/data_type.cc +++ b/mindspore/ccsrc/minddata/dataset/core/data_type.cc @@ -51,7 +51,9 @@ uint8_t DataType::AsCVType() const { } if (res == kCVInvalidType) { - MS_LOG(ERROR) << "Cannot convert to OpenCV type. Return invalid type!"; + std::string err_msg = "Cannot convert [" + std::string(kTypeInfo[type_].name_) + "] to OpenCV type."; + err_msg += " Currently unsupported data type: [uint32, int64, uint64, string]"; + MS_LOG(ERROR) << err_msg; } return res; diff --git a/mindspore/ccsrc/minddata/dataset/core/tensor.cc b/mindspore/ccsrc/minddata/dataset/core/tensor.cc index b2cb4ebdb0..32edfa451c 100644 --- a/mindspore/ccsrc/minddata/dataset/core/tensor.cc +++ b/mindspore/ccsrc/minddata/dataset/core/tensor.cc @@ -269,10 +269,11 @@ Status Tensor::CreateFromFile(const std::string &path, std::shared_ptr * CHECK_FAIL_RETURN_UNEXPECTED(!fs.fail(), "Fail to open file: " + path); int64_t num_bytes = fs.seekg(0, std::ios::end).tellg(); CHECK_FAIL_RETURN_UNEXPECTED(num_bytes <= kDeMaxDim, "Invalid file to allocate tensor memory, check path: " + path); - CHECK_FAIL_RETURN_UNEXPECTED(fs.seekg(0, std::ios::beg).good(), "Fail to find size of file"); + CHECK_FAIL_RETURN_UNEXPECTED(fs.seekg(0, std::ios::beg).good(), "Fail to find size of file, check path: " + path); RETURN_IF_NOT_OK(Tensor::CreateEmpty(TensorShape{num_bytes}, DataType(DataType::DE_UINT8), out)); int64_t written_bytes = fs.read(reinterpret_cast((*out)->GetMutableBuffer()), num_bytes).gcount(); - CHECK_FAIL_RETURN_UNEXPECTED(written_bytes == num_bytes && fs.good(), "Error in writing to tensor"); + CHECK_FAIL_RETURN_UNEXPECTED(written_bytes == num_bytes && fs.good(), + "Error in writing to tensor, check path: " + path); fs.close(); return Status::OK(); } @@ -873,6 +874,42 @@ Status Tensor::CopyLastDimAt(const std::shared_ptr &src, const std::vect return Status::OK(); } +Status Tensor::GetSliceOption(const SliceOption &slice_option, const int32_t &slice_index, + SliceOption *slice_option_ptr) { + if (slice_option.indices_.empty() && !slice_option.slice_.valid()) { + RETURN_STATUS_UNEXPECTED("Both indices and slices can not be empty."); + } + + if (!slice_option.indices_.empty() && slice_option.slice_.valid()) { + RETURN_STATUS_UNEXPECTED("Both indices and slices can not be given."); + } + + // if slice object was provided, indices should be empty. Generate indices from the slice object. + if (slice_option.indices_.empty()) { + // check if slice is valid + mindspore::dataset::Slice slice_copy = slice_option.slice_; + slice_copy.start_ = HandleNeg(slice_option.slice_.start_, shape_[slice_index]); + slice_copy.stop_ = HandleNeg(slice_option.slice_.stop_, shape_[slice_index]); + slice_copy.start_ = slice_copy.start_ < 0 ? 0 : slice_copy.start_; + slice_copy.stop_ = slice_copy.stop_ < 0 ? 0 : slice_copy.stop_; + dsize_t max_idx = shape_[slice_index]; + slice_copy.start_ = slice_copy.start_ > max_idx ? max_idx : slice_copy.start_; + slice_copy.stop_ = slice_copy.stop_ > max_idx ? max_idx : slice_copy.stop_; + *slice_option_ptr = SliceOption(slice_copy); + } else { + // indices validation + std::vector indices_copy; + for (int j = 0; j < slice_option.indices_.size(); j++) { + dsize_t index = HandleNeg(slice_option.indices_[j], shape_[slice_index]); + CHECK_FAIL_RETURN_UNEXPECTED(index < shape_[slice_index] && index >= 0, + "Index " + std::to_string(index) + " is out of bounds."); + indices_copy.emplace_back(index); + } + *slice_option_ptr = SliceOption(indices_copy); + } + return Status::OK(); +} + Status Tensor::Slice(std::shared_ptr *out, const std::vector slice_options_) { std::vector converted_slice_objects; @@ -885,37 +922,9 @@ Status Tensor::Slice(std::shared_ptr *out, const std::vector max_idx ? max_idx : slice_copy.start_; - slice_copy.stop_ = slice_copy.stop_ > max_idx ? max_idx : slice_copy.stop_; - converted_slice_objects.emplace_back(SliceOption(slice_copy)); - } else { - // indices validation - std::vector indices_copy; - for (int j = 0; j < slice_option.indices_.size(); j++) { - dsize_t index = HandleNeg(slice_option.indices_[j], shape_[i]); - CHECK_FAIL_RETURN_UNEXPECTED(index < shape_[i] && index >= 0, - "Index " + std::to_string(index) + " is out of bounds."); - indices_copy.emplace_back(index); - } - converted_slice_objects.emplace_back(SliceOption(indices_copy)); - } + SliceOption slice_option_item(false); + RETURN_IF_NOT_OK(GetSliceOption(slice_option, i, &slice_option_item)); + converted_slice_objects.emplace_back(slice_option_item); } // if a string with partial slices, pass in the rest diff --git a/mindspore/ccsrc/minddata/dataset/core/tensor.h b/mindspore/ccsrc/minddata/dataset/core/tensor.h index ac60324098..8603ce8ad2 100644 --- a/mindspore/ccsrc/minddata/dataset/core/tensor.h +++ b/mindspore/ccsrc/minddata/dataset/core/tensor.h @@ -405,6 +405,13 @@ class Tensor { /// \return Status error code Status Slice(TensorPtr *out, const std::vector slice_options); + /// Get slice_option according to shape and index. + /// \param[in] slice_option input SliceOption object + /// \param[in] slice_index index of SliceOption object + /// \param[out] output slice_option with shape info + /// \return Status error code + Status GetSliceOption(const SliceOption &slice_option, const int32_t &slice_index, SliceOption *slice_option_ptr); + #ifdef ENABLE_PYTHON /// Constructs numpy array from input tensor /// \param[in] data this data is the location of python data diff --git a/mindspore/ccsrc/minddata/dataset/include/tensor.h b/mindspore/ccsrc/minddata/dataset/include/tensor.h index c3e592ee6b..03be78fe1c 100644 --- a/mindspore/ccsrc/minddata/dataset/include/tensor.h +++ b/mindspore/ccsrc/minddata/dataset/include/tensor.h @@ -41,6 +41,7 @@ #include "include/api/status.h" #include "minddata/dataset/core/constants.h" #include "minddata/dataset/core/data_type.h" +#include "minddata/dataset/core/tensor_helpers.h" #include "minddata/dataset/core/tensor_shape.h" #include "minddata/dataset/core/de_tensor.h" #ifndef ENABLE_ANDROID @@ -171,6 +172,18 @@ class Tensor { return CreateFromVector(items, TensorShape({static_cast(items.size())}), out); } + /// Create a 1D boolean Tensor from a given list of boolean values. + /// \param[in] items elements of the tensor + /// \param[in] shape shape of the output tensor + /// \param[out] out output argument to hold the created Tensor + /// \return Status Code + static Status CreateFromVector(const std::vector &items, const TensorShape &shape, TensorPtr *out) { + std::vector temp(items.begin(), items.end()); + RETURN_IF_NOT_OK(CreateFromVector(temp, shape, out)); + (*out)->type_ = DataType(DataType::DE_BOOL); + return Status::OK(); + } + /// Create a numeric scalar Tensor from the given value. /// \tparam T type of value /// \param[in] item value @@ -282,7 +295,7 @@ class Tensor { const TensorShape &shape() const { return shape_; } /// Check if tensor has data - /// \return bool - true if tensor is empty + /// \return bool - true if tensor is not empty bool HasData() const { return data_ != nullptr; } /// Reshape the tensor. The given shape should have the same number of elements in the Tensor @@ -367,20 +380,37 @@ class Tensor { } /// Handle negative indices. + /// \param[out] out modified index + /// \param[in] index + /// \param[in] length axis length used to modify index + /// \return dsize_t modified index static inline dsize_t HandleNeg(dsize_t index, dsize_t length) { return (index < 0) ? (index + length) : index; } - /// Slice tensor bases on the given indices. Copy the sliced data into out tensor. Only rank1 tensors are supported. + /// Handle negative indices for a vector of indices. + /// \param[out] out modified vector of indices + /// \param[in] index_vector vector of indices + /// \return std::vector modified vector of indices + static inline std::vector HandleNegIndices(std::vector index_vector, std::vector length) { + std::vector indices(index_vector.size(), 0); + for (int i = 0; i < index_vector.size(); i++) { + indices[i] = HandleNeg(index_vector[i], length[i]); + } + return indices; + } + + /// Slice tensor bases on the given indices. Copy the sliced data into out tensor. /// Based on the type of tensor, SliceNumeric or SliceString will be called /// \param[out] out Tensor - /// \param[in] indices vector of indices + /// \param[in] slice_options vector of SliceOption objects /// \return Status error code - Status Slice(TensorPtr *out, const std::vector &indices); - - /// Slice numeric tensors. - Status SliceNumeric(TensorPtr *out, const std::vector &indices); + Status Slice(TensorPtr *out, const std::vector slice_options); - /// Slice string tensors - Status SliceString(TensorPtr *out, const std::vector &indices); + /// Get slice_option according to shape and index. + /// \param[in] slice_option input SliceOption object + /// \param[in] slice_index index of SliceOption object + /// \param[out] output slice_option with shape info + /// \return Status error code + Status GetSliceOption(const SliceOption &slice_option, const int32_t &slice_index, SliceOption *slice_option_ptr); #ifdef ENABLE_PYTHON /// Constructs numpy array from input tensor @@ -659,6 +689,12 @@ class Tensor { private: friend class DETensor; + /// Slice numeric tensors. + Status SliceNumeric(TensorPtr *out, const std::vector> &indices, const TensorShape &shape); + + /// Slice string tensors + Status SliceString(TensorPtr *out, const std::vector> &indices, const TensorShape &shape); + /// Copy raw data of a array based on shape and strides to the destination pointer /// \param dst [out] Pointer to the destination array where the content is to be copied /// \param[in] src Pointer to the source of strided array to be copied diff --git a/mindspore/dataset/vision/py_transforms_util.py b/mindspore/dataset/vision/py_transforms_util.py index f9983d9d67..7e05e5e983 100644 --- a/mindspore/dataset/vision/py_transforms_util.py +++ b/mindspore/dataset/vision/py_transforms_util.py @@ -56,6 +56,10 @@ def normalize(img, mean, std, pad_channel=False, dtype="float32"): Returns: img (numpy.ndarray), Normalized image. """ + if np.issubdtype(img.dtype, np.integer): + raise NotImplementedError("Unsupported image datatype: [{}], pls execute [ToTensor] before [Normalize]." + .format(img.dtype)) + if not is_numpy(img): raise TypeError("img should be NumPy image. Got {}.".format(type(img))) @@ -74,6 +78,7 @@ def normalize(img, mean, std, pad_channel=False, dtype="float32"): mean = np.array(mean, dtype=img.dtype) std = np.array(std, dtype=img.dtype) + image = (img - mean[:, None, None]) / std[:, None, None] if pad_channel: zeros = np.zeros([1, image.shape[1], image.shape[2]], dtype=np.float32)