- Add checks and testing for empty tensors

- cleanup work on createTensor and Tensor's constructors
pull/2983/head
hesham 5 years ago
parent d8fcf269d6
commit adfbc891d3

@ -511,8 +511,9 @@ Status DEPipeline::FetchDataFromTensorRow(const TensorRow &row,
RETURN_IF_NOT_OK(s);
if (data != nullptr) (*row_raw_data)[column_name] = std::move(*data);
} else if (column_type == DataType::DE_STRING) {
auto buffer = tensor->GetStringsBuffer();
std::string ss(reinterpret_cast<const char *>(buffer)); // assume scalar string tensor
std::string_view sv;
RETURN_IF_NOT_OK(tensor->GetItemAt(&sv, {0})); // assume scalar string tensor
std::string ss(sv);
(*row_raw_data)[column_name] = std::move(ss);
continue;
} else {
@ -1678,13 +1679,13 @@ Status DEPipeline::ParsePadInfo(py::handle value, PadInfo *pad_info) {
if (py::isinstance<py::str>(tp[1])) {
std::string pad_val_string = tp[1].is_none() ? "" : ToString(tp[1]);
CHECK_FAIL_RETURN_UNEXPECTED(
Tensor::CreateTensor(&pad_val, std::vector<std::string>{pad_val_string}, TensorShape::CreateScalar()),
Tensor::CreateFromVector(std::vector<std::string>{pad_val_string}, TensorShape::CreateScalar(), &pad_val),
"Cannot create pad_value Tensor");
} else {
float pad_val_float = tp[1].is_none() ? 0 : ToFloat(tp[1]);
CHECK_FAIL_RETURN_UNEXPECTED(Tensor::CreateTensor(&pad_val, TensorImpl::kFlexible, TensorShape::CreateScalar(),
DataType(DataType::DE_FLOAT32)),
"Cannot create pad_value Tensor");
CHECK_FAIL_RETURN_UNEXPECTED(
Tensor::CreateEmpty(TensorShape::CreateScalar(), DataType(DataType::DE_FLOAT32), &pad_val),
"Cannot create pad_value Tensor");
pad_val->SetItemAt<float>({}, pad_val_float);
}
(void)pad_info->insert({ToString(p.first), {shape, pad_val}});

@ -340,7 +340,7 @@ void bindTensor(py::module *m) {
(void)py::class_<Tensor, std::shared_ptr<Tensor>>(*m, "Tensor", py::buffer_protocol())
.def(py::init([](py::array arr) {
std::shared_ptr<Tensor> out;
THROW_IF_ERROR(Tensor::CreateTensor(&out, arr));
THROW_IF_ERROR(Tensor::CreateFromNpArray(arr, &out));
return out;
}))
.def_buffer([](Tensor &tensor) {
@ -364,7 +364,18 @@ void bindTensor(py::module *m) {
});
(void)py::class_<TensorShape>(*m, "TensorShape")
.def(py::init<py::list>())
.def(py::init([](const py::list &list) {
std::vector<dsize_t> list_c;
for (auto &i : list) {
if (!i.is_none()) {
list_c.push_back(i.cast<int>());
} else {
list_c.push_back(TensorShape::kDimUnknown);
}
}
TensorShape out(list_c);
return out;
}))
.def("__str__", &TensorShape::ToString)
.def("as_list", &TensorShape::AsPyList)
.def("is_known", &TensorShape::known);

@ -23,16 +23,33 @@
namespace mindspore {
namespace dataset {
CVTensor::CVTensor(const TensorShape &shape, const DataType &type) : Tensor(shape, type) {
CVTensor::CVTensor(std::shared_ptr<Tensor> tensor) : Tensor(std::move(*tensor)) {
(void)this->MatInit(GetMutableBuffer(), shape_, type_, &mat_);
}
CVTensor::CVTensor(const TensorShape &shape, const DataType &type, const uchar *data) : Tensor(shape, type, data) {
(void)this->MatInit(GetMutableBuffer(), shape_, type_, &mat_);
Status CVTensor::CreateEmpty(const TensorShape &shape, DataType type, CVTensorPtr *out) {
const CVTensorAlloc *alloc = GlobalContext::Instance()->cv_tensor_allocator();
*out = std::allocate_shared<CVTensor>(*alloc, shape, type);
int64_t byte_size = (*out)->SizeInBytes();
// Don't allocate if we have a tensor with no elements.
if (byte_size != 0) {
RETURN_IF_NOT_OK((*out)->AllocateBuffer(byte_size));
}
return (*out)->MatInit((*out)->GetMutableBuffer(), (*out)->shape_, (*out)->type_, &(*out)->mat_);
}
CVTensor::CVTensor(std::shared_ptr<Tensor> tensor) : Tensor(std::move(*tensor)) {
(void)this->MatInit(GetMutableBuffer(), shape_, type_, &mat_);
Status CVTensor::CreateFromMat(const cv::Mat &mat, CVTensorPtr *out) {
TensorPtr out_tensor;
cv::Mat mat_local = mat;
// if the input Mat's memory is not continuous, copy it to one block of memory
if (!mat.isContinuous()) mat_local = mat.clone();
TensorShape shape(mat.size, mat_local.type());
DataType type = DataType::FromCVType(mat_local.type());
RETURN_IF_NOT_OK(CreateFromMemory(shape, type, mat_local.data, &out_tensor));
*out = AsCVTensor(out_tensor);
return Status::OK();
}
std::pair<std::array<int, 2>, int> CVTensor::IsValidImage(const TensorShape &shape, const DataType &type) {
@ -57,7 +74,8 @@ std::shared_ptr<CVTensor> CVTensor::AsCVTensor(std::shared_ptr<Tensor> t) {
if (cv_t != nullptr) {
return cv_t;
} else {
return std::make_shared<CVTensor>(t);
const CVTensorAlloc *alloc = GlobalContext::Instance()->cv_tensor_allocator();
return std::allocate_shared<CVTensor>(*alloc, t);
}
}
@ -97,5 +115,13 @@ void CVTensor::Squeeze() {
Tensor::Squeeze();
(void)this->MatInit(GetMutableBuffer(), shape_, type_, &mat_);
}
Status CVTensor::MatAtIndex(const std::vector<dsize_t> &index, cv::Mat *mat) {
uchar *start = nullptr;
TensorShape remaining({-1});
RETURN_IF_NOT_OK(this->StartAddrOfIndex(index, &start, &remaining));
RETURN_IF_NOT_OK(this->MatInit(start, remaining, type_, mat));
return Status::OK();
}
} // namespace dataset
} // namespace mindspore

@ -30,56 +30,60 @@
namespace mindspore {
namespace dataset {
using CVTensorPtr = std::shared_ptr<CVTensor>;
class CVTensor : public Tensor {
public:
// Create an empty CVTensor of shape `shape` and type `type`.
// @note The shape and type information should be known and valid.
// @param shape TensorShape
// @param type DataType
CVTensor(const TensorShape &shape, const DataType &type);
// Create a CVTensor from a given buffer, shape and type.
// @note This constructor allocates a new space in the memory and copies the buffer into it.
// @note The buffer should be valid and the shape and type information should be known and valid.
// @param shape TensorShape
// @param type DataType
// @param data unsigned char*, pointer to the data.
CVTensor(const TensorShape &shape, const DataType &type, const uchar *data);
// Create a CVTensor from a given CV::Mat.
// @note This constructor allocates a new space in the memory and copies the CV::Mat buffer into it.
// @param mat CV::Mat
explicit CVTensor(const cv::Mat &mat)
: CVTensor(TensorShape(mat.size, mat.type()), DataType::FromCVType(mat.type()), mat.data) {}
~CVTensor() = default;
// Static function to cast a given Tensor as CVTensor. If the input tensor is already of type CVTensor,
// this function would be treated as a no-op. Fot other tensor types, a new CVTensor is created based on the data
// provided. The Passed Tensor will be invalidated.
// @note there is no memory copying here, the buffer will be assigned to the constructed tensor.
// @param tensor
// @return CVTensor
static std::shared_ptr<CVTensor> AsCVTensor(std::shared_ptr<Tensor> tensor);
// Create a CVTensor from a given tensor. The input tensor will be invalidated (i.e., the shape and type will be
// set to unknown and the data buffer will point to null.
// @note there is no memory copying here, the buffer will be assigned to the constructed tensor.
// @param tensor
// Inherit Tensor's constructors
using Tensor::Tensor;
/// Create a CVTensor from a given tensor. This constructor should not be used directly, use Create* instead.
/// The input tensor will be invalidated (i.e., the shape and type will be
/// set to unknown and the data buffer will point to null.
/// \note there is no memory copying here, the buffer will be assigned to the constructed tensor.
/// \param tensor
explicit CVTensor(std::shared_ptr<Tensor> tensor);
// Getter function for the CV::Mat
// @return
/// Create CV tensor with type and shape. Items of the tensor would be uninitialized.
/// \param shape [in] shape of the output tensor
/// \param type [in] type of the output tensor
/// \param out [out] Generated tensor
/// \return Status code
static Status CreateEmpty(const TensorShape &shape, DataType type, CVTensorPtr *out);
/// Create CV tensor from cv::Mat
/// \note This constructor allocates a new space in the memory and copies the CV::Mat buffer into it.
/// \param mat [in] cv::Mat to be copied into the new tensor.
/// \param out [out] Generated tensor
/// \return Status code
static Status CreateFromMat(const cv::Mat &mat, CVTensorPtr *out);
~CVTensor() override = default;
/// Static function to cast a given Tensor as CVTensor. If the input tensor is already of type CVTensor,
/// this function would be treated as a no-op. Fot other tensor types, a new CVTensor is created based on the data
/// provided. The Passed Tensor will be invalidated.
/// \note the input tensor will be invalidated.
/// \note there is no memory copying here, the buffer will be assigned to the constructed tensor.
/// \param tensor [in]
/// \return CVTensor
static std::shared_ptr<CVTensor> AsCVTensor(std::shared_ptr<Tensor> tensor);
/// Get a reference to the CV::Mat
/// \return a reference to the internal CV::Mat
cv::Mat mat() const { return mat_; }
// Static function to check if the passed information (shape and type) can be treated as a valid description
// of an image in OpenCV. Moreover, it returns OpenCV shape and type
// For example, if the shape is <512,512,3> and type is DE_UINT8, the output would be [512,512] and CV_8UC3.
// In case of invalid shape or type, the function will return pair<null,0>
// @param shape TensorShape
// @param type DataType
// @return std::pair of OpenCV shape and type
std::pair<std::array<int, 2>, int> IsValidImage(const TensorShape &shape, const DataType &type);
/// Get a copy of the CV::Mat
/// \return a copy of internal CV::Mat
cv::Mat matCopy() const { return mat_.clone(); }
/// Static function to check if the passed information (shape and type) can be treated as a valid description
/// of an image in OpenCV. Moreover, it returns OpenCV shape and type
/// For example, if the shape is <512,512,3> and type is DE_UINT8, the output would be [512,512] and CV_8UC3.
/// In case of invalid shape or type, the function will return pair<null,0>
/// \param shape [in] TensorShape
/// \param type [in] DataType
/// \return std::pair of OpenCV shape and type
static std::pair<std::array<int, 2>, int> IsValidImage(const TensorShape &shape, const DataType &type);
Status Reshape(const TensorShape &shape) override;
@ -87,18 +91,19 @@ class CVTensor : public Tensor {
void Squeeze() override;
Status Mat(const std::vector<dsize_t> &index, cv::Mat *mat) {
uchar *start = nullptr;
TensorShape remaining({-1});
RETURN_IF_NOT_OK(this->StartAddrOfIndex(index, &start, &remaining));
RETURN_IF_NOT_OK(this->MatInit(start, remaining, type_, mat));
return Status::OK();
}
Status MatAtIndex(const std::vector<dsize_t> &index, cv::Mat *mat);
private:
/// Opencv Mat object wrapping the raw data of the tensor.
/// Modifying the content of the matrix, modifies the tensor.
cv::Mat mat_;
// Initialize CV::Mat with the data_, shape_ and type_
/// Create cv::Mat from data, TensorShape and DataType
/// \param data [in] Pointer to the data in memory.
/// \param shape [in] Shape of the tensor.
/// \param type [in] Type of the tensor.
/// \param mat [out] cv::Mat initialized with the provided data.
/// \return Status code
Status MatInit(uchar *data, const TensorShape &shape, const DataType &type, cv::Mat *mat);
};
} // namespace dataset

@ -284,6 +284,11 @@ inline DataType DataType::FromCType<std::string_view>() {
return DataType(DataType::DE_STRING);
}
template <>
inline DataType DataType::FromCType<std::string>() {
return DataType(DataType::DE_STRING);
}
template <>
inline bool DataType::IsLooselyCompatible<bool>() const {
return type_ == DataType::DE_BOOL;

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

@ -141,8 +141,9 @@ Status BatchFetchRequest::RestoreOneTensor(const TensorMetaMsg *col_ts, const Re
#undef CASE
DataType type(dest);
std::shared_ptr<Tensor> ts =
std::make_shared<Tensor>(shape, type, static_cast<const unsigned char *>(data.GetPointer()), data.GetSize());
std::shared_ptr<Tensor> ts;
RETURN_IF_NOT_OK(
Tensor::CreateFromMemory(shape, type, static_cast<const unsigned char *>(data.GetPointer()), data.GetSize(), &ts));
// Next we restore the real data which can be embedded or stored separately.
if (ts->SizeInBytes() != data.GetSize()) {
MS_LOG(ERROR) << "Unexpected length. Read " << data.GetSize() << ". Expected " << ts->SizeInBytes() << ".\n"

@ -176,12 +176,15 @@ Status BatchOp::BatchRows(const std::unique_ptr<TensorQTable> *src, const std::u
std::shared_ptr<Tensor> new_tensor;
if (first_type.IsNumeric()) { // numeric tensor
RETURN_IF_NOT_OK(Tensor::CreateTensor(&new_tensor, TensorImpl::kFlexible, new_shape, first_type));
RETURN_IF_NOT_OK(Tensor::CreateEmpty(new_shape, first_type, &new_tensor));
dsize_t j = 0;
for (auto row : **src) {
std::shared_ptr<Tensor> old_tensor = row.at(i); // row j, column i
if (old_tensor->shape() == first_shape) { // check the newly popped rows have the same dim as the first
RETURN_IF_NOT_OK(new_tensor->InsertTensor({j++}, old_tensor));
if (new_shape.NumOfElements() != 0) {
RETURN_IF_NOT_OK(new_tensor->InsertTensor({j++}, old_tensor));
}
// Don't do anything if the tensor has no data
} else {
RETURN_STATUS_UNEXPECTED("[Batch ERROR] Inconsistent TensorShapes of Column " + std::to_string(i));
}
@ -194,7 +197,7 @@ Status BatchOp::BatchRows(const std::unique_ptr<TensorQTable> *src, const std::u
strings.emplace_back(*itr);
}
}
RETURN_IF_NOT_OK(Tensor::CreateTensor(&new_tensor, strings, new_shape));
RETURN_IF_NOT_OK(Tensor::CreateFromVector(strings, new_shape, &new_tensor));
}
batched_row.emplace_back(new_tensor);
}
@ -352,7 +355,7 @@ Status BatchOp::InvokeBatchMapFunc(TensorBatchTable *input, TensorBatchTable *ou
py::list output_list = py::cast<py::list>(ret_tuple[i]);
for (size_t j = 0; j < output_list.size(); j++) {
std::shared_ptr<Tensor> out;
RETURN_IF_NOT_OK(Tensor::CreateTensor(&out, py::cast<py::array>(output_list[j])));
RETURN_IF_NOT_OK(Tensor::CreateFromNpArray(py::cast<py::array>(output_list[j]), &out));
output_batch.push_back(std::move(out));
}
output->push_back(std::move(output_batch));

@ -226,7 +226,8 @@ void CacheMergeOp::TensorRowRequest::WakeUpAny(TensorRow &&row) {
if (GetState() == State::kEmpty) {
// We will do a deep copy
for (auto &ts : row) {
auto out_ts = std::make_shared<Tensor>(ts->shape(), ts->type(), ts->GetBuffer(), ts->SizeInBytes());
std::shared_ptr<Tensor> out_ts;
Tensor::CreateFromTensor(ts, &out_ts);
cleaner_copy_.push_back(out_ts);
}
cleaner_copy_.setId(row.getId());

@ -72,6 +72,7 @@ Status DeviceQueueOp::CheckExceptions(const std::unique_ptr<DataBuffer> &buffer)
buffer->GetRow(0, &row);
for (const auto &item : row) {
CHECK_FAIL_RETURN_UNEXPECTED(item->type().IsNumeric(), "Cannot send tensor of string type to device.");
CHECK_FAIL_RETURN_UNEXPECTED(item->HasData(), "Cannot send tensor with no data.");
}
}
return Status::OK();

@ -359,7 +359,7 @@ Status CelebAOp::LoadTensorRow(row_id_type row_id, const std::pair<std::string,
Path path(folder_path_);
Path image_path = path / image_label.first;
RETURN_IF_NOT_OK(Tensor::CreateTensor(&image, image_path.toString()));
RETURN_IF_NOT_OK(Tensor::CreateFromFile(image_path.toString(), &image));
if (decode_ == true) {
Status rc = Decode(image, &image);
if (rc.IsError()) {
@ -369,9 +369,8 @@ Status CelebAOp::LoadTensorRow(row_id_type row_id, const std::pair<std::string,
}
}
RETURN_IF_NOT_OK(Tensor::CreateTensor(&label, data_schema_->column(1).tensorImpl(),
TensorShape({1, (uint32_t)image_label.second.size()}),
data_schema_->column(1).type()));
RETURN_IF_NOT_OK(
Tensor::CreateEmpty(TensorShape({1, (uint32_t)image_label.second.size()}), data_schema_->column(1).type(), &label));
RETURN_IF_NOT_OK(label->Zero());
for (uint32_t index = 0; index < image_label.second.size(); index++) {
if (image_label.second[index] == 1) {

@ -190,15 +190,12 @@ Status CifarOp::LoadTensorRow(uint64_t index, TensorRow *trow) {
std::shared_ptr<Tensor> label;
std::shared_ptr<Tensor> fine_label;
std::shared_ptr<Tensor> ori_image = cifar_image_label_pairs_[index].first;
std::shared_ptr<Tensor> copy_image =
std::make_shared<Tensor>(ori_image->shape(), ori_image->type(), ori_image->GetBuffer());
RETURN_IF_NOT_OK(Tensor::CreateTensor(&label, data_schema_->column(1).tensorImpl(), data_schema_->column(1).shape(),
data_schema_->column(1).type(),
reinterpret_cast<unsigned char *>(&cifar_image_label_pairs_[index].second[0])));
std::shared_ptr<Tensor> copy_image;
RETURN_IF_NOT_OK(Tensor::CreateFromTensor(ori_image, &copy_image));
RETURN_IF_NOT_OK(Tensor::CreateScalar(cifar_image_label_pairs_[index].second[0], &label));
if (cifar_image_label_pairs_[index].second.size() > 1) {
RETURN_IF_NOT_OK(Tensor::CreateTensor(
&fine_label, data_schema_->column(2).tensorImpl(), data_schema_->column(2).shape(),
data_schema_->column(2).type(), reinterpret_cast<unsigned char *>(&cifar_image_label_pairs_[index].second[1])));
RETURN_IF_NOT_OK(Tensor::CreateScalar(cifar_image_label_pairs_[index].second[1], &fine_label));
(*trow) = TensorRow(index, {copy_image, std::move(label), std::move(fine_label)});
} else {
(*trow) = TensorRow(index, {copy_image, std::move(label)});
@ -359,9 +356,8 @@ Status CifarOp::ParseCifarData() {
}
std::shared_ptr<Tensor> image_tensor;
RETURN_IF_NOT_OK(Tensor::CreateTensor(&image_tensor, data_schema_->column(0).tensorImpl(),
TensorShape({kCifarImageHeight, kCifarImageWidth, kCifarImageChannel}),
data_schema_->column(0).type()));
RETURN_IF_NOT_OK(Tensor::CreateEmpty(TensorShape({kCifarImageHeight, kCifarImageWidth, kCifarImageChannel}),
data_schema_->column(0).type(), &image_tensor));
auto itr = image_tensor->begin<uint8_t>();
uint32_t total_pix = kCifarImageHeight * kCifarImageWidth;
for (int pix = 0; pix < total_pix; ++pix) {

@ -127,7 +127,7 @@ Status ClueOp::LoadTensor(const std::string &line, std::unique_ptr<TensorQTable>
(*tensor_table)->push_back(std::move(tRow));
std::shared_ptr<Tensor> tensor;
RETURN_IF_NOT_OK(Tensor::CreateTensor(&tensor, {line}, TensorShape::CreateScalar()));
RETURN_IF_NOT_OK(Tensor::CreateScalar(line, &tensor));
(**tensor_table)[row][0] = std::move(tensor);
return Status::OK();
}
@ -144,26 +144,19 @@ Status ClueOp::GetValue(const nlohmann::json &js, std::vector<std::string> key_c
std::string final_str = key_chain.back();
switch (cursor.type()) {
case nlohmann::detail::value_t::string:
RETURN_IF_NOT_OK(Tensor::CreateTensor(t, {cursor.get<std::string>()}, TensorShape::CreateScalar()));
RETURN_IF_NOT_OK(Tensor::CreateScalar(cursor.get<std::string>(), t));
break;
case nlohmann::detail::value_t::number_integer:
RETURN_IF_NOT_OK(
Tensor::CreateTensor(t, TensorImpl::kFlexible, TensorShape::CreateScalar(), DataType(DataType::DE_INT32)));
(*t)->SetItemAt<int32_t>({0}, cursor.get<int32_t>());
RETURN_IF_NOT_OK(Tensor::CreateScalar(cursor.get<int32_t>(), t));
break;
case nlohmann::detail::value_t::number_unsigned:
RETURN_IF_NOT_OK(
Tensor::CreateTensor(t, TensorImpl::kFlexible, TensorShape::CreateScalar(), DataType(DataType::DE_INT32)));
(*t)->SetItemAt<int32_t>({0}, cursor.get<uint32_t>());
RETURN_IF_NOT_OK(Tensor::CreateScalar(cursor.get<uint32_t>(), t));
break;
case nlohmann::detail::value_t::number_float:
RETURN_IF_NOT_OK(
Tensor::CreateTensor(t, TensorImpl::kFlexible, TensorShape::CreateScalar(), DataType(DataType::DE_FLOAT32)));
(*t)->SetItemAt<int32_t>({0}, cursor.get<float>());
RETURN_IF_NOT_OK(Tensor::CreateScalar(cursor.get<float>(), t));
break;
case nlohmann::detail::value_t::array:
RETURN_IF_NOT_OK(Tensor::CreateTensor(t, {cursor.get<std::vector<std::string>>()}, TensorShape::CreateScalar()));
RETURN_IF_NOT_OK(Tensor::CreateFromVector(cursor.get<std::vector<std::string>>(), t));
break;
default:
break;

@ -239,9 +239,8 @@ Status CocoOp::LoadTensorRow(row_id_type row_id, const std::string &image_id, Te
}
std::vector<dsize_t> bbox_dim = {bbox_row_num, bbox_column_num};
RETURN_IF_NOT_OK(Tensor::CreateTensor(&coordinate, data_schema_->column(1).tensorImpl(), TensorShape(bbox_dim),
data_schema_->column(1).type(),
reinterpret_cast<unsigned char *>(&bbox_row[0])));
RETURN_IF_NOT_OK(Tensor::CreateFromVector(bbox_row, TensorShape(bbox_dim), &coordinate));
if (task_type_ == TaskType::Detection) {
RETURN_IF_NOT_OK(LoadDetectionTensorRow(row_id, image_id, image, coordinate, trow));
} else if (task_type_ == TaskType::Stuff || task_type_ == TaskType::Keypoint) {
@ -278,13 +277,12 @@ Status CocoOp::LoadDetectionTensorRow(row_id_type row_id, const std::string &ima
iscrowd_row.push_back(annotation[i]);
}
}
RETURN_IF_NOT_OK(Tensor::CreateTensor(
&category_id, data_schema_->column(2).tensorImpl(), TensorShape({static_cast<dsize_t>(category_id_row.size()), 1}),
data_schema_->column(2).type(), reinterpret_cast<unsigned char *>(&category_id_row[0])));
RETURN_IF_NOT_OK(Tensor::CreateFromVector(
category_id_row, TensorShape({static_cast<dsize_t>(category_id_row.size()), 1}), &category_id));
RETURN_IF_NOT_OK(
Tensor::CreateFromVector(iscrowd_row, TensorShape({static_cast<dsize_t>(iscrowd_row.size()), 1}), &iscrowd));
RETURN_IF_NOT_OK(Tensor::CreateTensor(
&iscrowd, data_schema_->column(3).tensorImpl(), TensorShape({static_cast<dsize_t>(iscrowd_row.size()), 1}),
data_schema_->column(3).type(), reinterpret_cast<unsigned char *>(&iscrowd_row[0])));
(*trow) = TensorRow(row_id, {std::move(image), std::move(coordinate), std::move(category_id), std::move(iscrowd)});
return Status::OK();
}
@ -302,9 +300,8 @@ Status CocoOp::LoadSimpleTensorRow(row_id_type row_id, const std::string &image_
item_queue = itr_item->second;
std::vector<dsize_t> bbox_dim = {static_cast<dsize_t>(item_queue.size()), 1};
RETURN_IF_NOT_OK(Tensor::CreateTensor(&item, data_schema_->column(2).tensorImpl(), TensorShape(bbox_dim),
data_schema_->column(2).type(),
reinterpret_cast<unsigned char *>(&item_queue[0])));
RETURN_IF_NOT_OK(Tensor::CreateFromVector(item_queue, TensorShape(bbox_dim), &item));
(*trow) = TensorRow(row_id, {std::move(image), std::move(coordinate), std::move(item)});
return Status::OK();
}
@ -334,18 +331,14 @@ Status CocoOp::LoadMixTensorRow(row_id_type row_id, const std::string &image_id,
area_row.push_back(annotation[i]);
}
}
RETURN_IF_NOT_OK(Tensor::CreateFromVector(
category_id_row, TensorShape({static_cast<dsize_t>(category_id_row.size()), 1}), &category_id));
RETURN_IF_NOT_OK(Tensor::CreateTensor(
&category_id, data_schema_->column(2).tensorImpl(), TensorShape({static_cast<dsize_t>(category_id_row.size()), 1}),
data_schema_->column(2).type(), reinterpret_cast<unsigned char *>(&category_id_row[0])));
RETURN_IF_NOT_OK(
Tensor::CreateFromVector(iscrowd_row, TensorShape({static_cast<dsize_t>(iscrowd_row.size()), 1}), &iscrowd));
RETURN_IF_NOT_OK(Tensor::CreateTensor(
&iscrowd, data_schema_->column(3).tensorImpl(), TensorShape({static_cast<dsize_t>(iscrowd_row.size()), 1}),
data_schema_->column(3).type(), reinterpret_cast<unsigned char *>(&iscrowd_row[0])));
RETURN_IF_NOT_OK(Tensor::CreateFromVector(area_row, TensorShape({static_cast<dsize_t>(area_row.size()), 1}), &area));
RETURN_IF_NOT_OK(Tensor::CreateTensor(
&area, data_schema_->column(4).tensorImpl(), TensorShape({static_cast<dsize_t>(area_row.size()), 1}),
data_schema_->column(4).type(), reinterpret_cast<unsigned char *>(&area_row[0])));
(*trow) = TensorRow(
row_id, {std::move(image), std::move(coordinate), std::move(category_id), std::move(iscrowd), std::move(area)});
return Status::OK();
@ -596,7 +589,7 @@ Status CocoOp::LaunchThreadsAndInitOp() {
}
Status CocoOp::ReadImageToTensor(const std::string &path, const ColDescriptor &col, std::shared_ptr<Tensor> *tensor) {
RETURN_IF_NOT_OK(Tensor::CreateTensor(tensor, path));
RETURN_IF_NOT_OK(Tensor::CreateFromFile(path, tensor));
if (decode_ == true) {
Status rc = Decode(*tensor, tensor);

@ -102,18 +102,13 @@ int CsvOp::CsvParser::put_record(char c) {
std::shared_ptr<Tensor> t;
switch (column_default_[cur_col_]->type) {
case CsvOp::INT:
Tensor::CreateTensor(&t, TensorImpl::kFlexible, TensorShape::CreateScalar(), DataType(DataType::DE_INT32));
t->SetItemAt<int32_t>({0}, std::stoi(s));
Tensor::CreateScalar(std::stoi(s), &t);
break;
case CsvOp::FLOAT:
Tensor::CreateTensor(&t, TensorImpl::kFlexible, TensorShape::CreateScalar(), DataType(DataType::DE_FLOAT32));
t->SetItemAt<float>({0}, std::stof(s));
break;
case CsvOp::STRING:
Tensor::CreateTensor(&t, {s}, TensorShape::CreateScalar());
Tensor::CreateScalar(std::stof(s), &t);
break;
default:
Tensor::CreateTensor(&t, {s}, TensorShape::CreateScalar());
Tensor::CreateScalar(s, &t);
break;
}
(*tensor_table_)[cur_row_][cur_col_] = std::move(t);

@ -129,7 +129,7 @@ Status GeneratorOp::PyRowToTensorRow(py::object py_data, TensorRow *tensor_row)
"Generator should return a tuple of numpy arrays.");
}
std::shared_ptr<Tensor> tensor;
RETURN_IF_NOT_OK(Tensor::CreateTensor(&tensor, ret_py_ele.cast<py::array>()));
RETURN_IF_NOT_OK(Tensor::CreateFromNpArray(ret_py_ele.cast<py::array>(), &tensor));
if ((!column_types_.empty()) && (column_types_[i] != DataType::DE_UNKNOWN) &&
(column_types_[i] != tensor->type())) {
return Status(StatusCode::kPyFuncException, __LINE__, __FILE__, "Generator type check failed.");

@ -201,10 +201,8 @@ Status ImageFolderOp::WorkerEntry(int32_t worker_id) {
// Load 1 TensorRow (image,label) using 1 ImageLabelPair. 1 function call produces 1 TensorTow in a DataBuffer
Status ImageFolderOp::LoadTensorRow(row_id_type row_id, ImageLabelPair pairPtr, TensorRow *trow) {
std::shared_ptr<Tensor> image, label;
RETURN_IF_NOT_OK(Tensor::CreateTensor(&label, data_schema_->column(1).tensorImpl(), data_schema_->column(1).shape(),
data_schema_->column(1).type(),
reinterpret_cast<unsigned char *>(&pairPtr->second)));
RETURN_IF_NOT_OK(Tensor::CreateTensor(&image, folder_path_ + (pairPtr->first)));
RETURN_IF_NOT_OK(Tensor::CreateScalar(pairPtr->second, &label));
RETURN_IF_NOT_OK(Tensor::CreateFromFile(folder_path_ + (pairPtr->first), &image));
if (decode_ == true) {
Status rc = Decode(image, &image);

@ -185,17 +185,14 @@ Status ManifestOp::LoadTensorRow(row_id_type row_id, const std::pair<std::string
std::vector<int32_t> label_index(data.second.size());
(void)std::transform(data.second.begin(), data.second.end(), label_index.begin(),
[this](const std::string &label_name) { return label_index_[label_name]; });
RETURN_IF_NOT_OK(Tensor::CreateFromVector(label_index, &label));
if (label_index.size() == 1) {
RETURN_IF_NOT_OK(Tensor::CreateTensor(&label, data_schema_->column(1).tensorImpl(), TensorShape({}),
data_schema_->column(1).type(),
reinterpret_cast<unsigned char *>(&label_index[0])));
label->Reshape(TensorShape({}));
} else {
RETURN_IF_NOT_OK(Tensor::CreateTensor(
&label, data_schema_->column(1).tensorImpl(), TensorShape(std::vector<dsize_t>(1, label_index.size())),
data_schema_->column(1).type(), reinterpret_cast<unsigned char *>(&label_index[0])));
label->Reshape(TensorShape(std::vector<dsize_t>(1, label_index.size())));
}
RETURN_IF_NOT_OK(Tensor::CreateTensor(&image, data.first));
RETURN_IF_NOT_OK(Tensor::CreateFromFile(data.first, &image));
if (decode_ == true) {
Status rc = Decode(image, &image);
if (rc.IsError()) {

@ -381,15 +381,15 @@ Status MindRecordOp::LoadTensorRow(TensorRow *tensor_row, const std::vector<uint
auto num_elements = n_bytes / column_data_type_size;
if (type == DataType::DE_STRING) {
std::string s{data, data + n_bytes};
RETURN_IF_NOT_OK(Tensor::CreateTensor(&tensor, {s}, TensorShape::CreateScalar()));
RETURN_IF_NOT_OK(Tensor::CreateScalar(s, &tensor));
} else if (column.hasShape()) {
auto new_shape = TensorShape(column.shape());
RETURN_IF_NOT_OK(column.MaterializeTensorShape(static_cast<int32_t>(num_elements), &new_shape));
RETURN_IF_NOT_OK(Tensor::CreateTensor(&tensor, column.tensorImpl(), new_shape, type, data));
RETURN_IF_NOT_OK(Tensor::CreateFromMemory(new_shape, type, data, &tensor));
} else {
std::vector<dsize_t> shapeDetails = {static_cast<dsize_t>(num_elements)};
auto new_shape = TensorShape(shapeDetails);
RETURN_IF_NOT_OK(Tensor::CreateTensor(&tensor, column.tensorImpl(), new_shape, type, data));
RETURN_IF_NOT_OK(Tensor::CreateFromMemory(new_shape, type, data, &tensor));
}
tensor_row->push_back(std::move(tensor));
}

@ -160,12 +160,10 @@ Status MnistOp::WorkerEntry(int32_t worker_id) {
// Load 1 TensorRow (image,label) using 1 MnistLabelPair.
Status MnistOp::LoadTensorRow(row_id_type row_id, const MnistLabelPair &mnist_pair, TensorRow *trow) {
std::shared_ptr<Tensor> image, label;
int32_t l = mnist_pair.second;
// make a copy of cached tensor
RETURN_IF_NOT_OK(Tensor::CreateTensor(&image, data_schema_->column(0).tensorImpl(), mnist_pair.first->shape(),
mnist_pair.first->type(), mnist_pair.first->GetBuffer()));
RETURN_IF_NOT_OK(Tensor::CreateTensor(&label, data_schema_->column(1).tensorImpl(), data_schema_->column(1).shape(),
data_schema_->column(1).type(), reinterpret_cast<unsigned char *>(&l)));
RETURN_IF_NOT_OK(Tensor::CreateFromTensor(mnist_pair.first, &image));
RETURN_IF_NOT_OK(Tensor::CreateScalar(mnist_pair.second, &label));
(*trow) = TensorRow(row_id, {std::move(image), std::move(label)});
return Status::OK();
}
@ -325,8 +323,8 @@ Status MnistOp::ReadImageAndLabel(std::ifstream *image_reader, std::ifstream *la
pixels[m] = (pixels[m] == 0) ? 0 : 255;
}
std::shared_ptr<Tensor> image;
RETURN_IF_NOT_OK(Tensor::CreateTensor(&image, data_schema_->column(0).tensorImpl(), img_tensor_shape,
data_schema_->column(0).type(), reinterpret_cast<unsigned char *>(pixels)));
RETURN_IF_NOT_OK(Tensor::CreateFromMemory(img_tensor_shape, data_schema_->column(0).type(),
reinterpret_cast<unsigned char *>(pixels), &image));
image_label_pairs_.emplace_back(std::make_pair(image, labels_buf[j]));
}
return Status::OK();

@ -40,7 +40,7 @@ namespace dataset {
template <typename T>
class Queue;
using MnistLabelPair = std::pair<std::shared_ptr<Tensor>, int32_t>;
using MnistLabelPair = std::pair<std::shared_ptr<Tensor>, uint32_t>;
class MnistOp : public ParallelOp, public RandomAccessOp {
public:

@ -361,8 +361,7 @@ Status RandomDataOp::CreateRandomRow(int32_t worker_id, TensorRow *new_row) {
return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, "Failed to set random bytes for a tensor.");
}
RETURN_IF_NOT_OK(
Tensor::CreateTensor(&new_tensor, current_col.tensorImpl(), *new_shape, current_col.type(), buf.get()));
RETURN_IF_NOT_OK(Tensor::CreateFromMemory(*new_shape, current_col.type(), buf.get(), &new_tensor));
// Add this tensor to the tensor row for output
(*new_row).push_back(std::move(new_tensor));

@ -41,7 +41,7 @@ Status PythonSampler::GetNextSample(std::unique_ptr<DataBuffer> *out_buffer) {
try {
py::object py_ret = py_sampler_instance.attr("_get_indices")();
py::array np_sample_ids = py_ret.cast<py::array>();
Tensor::CreateTensor(&sample_ids, np_sample_ids); // copy numpy to tensor
Tensor::CreateFromNpArray(np_sample_ids, &sample_ids); // copy numpy to tensor
if (HasChildSampler()) {
for (auto it = sample_ids->begin<int64_t>(); it != sample_ids->end<int64_t>(); ++it) {

@ -73,9 +73,7 @@ Status Sampler::CreateSamplerTensor(std::shared_ptr<Tensor> *sample_ids, int64_t
col_desc_ = std::make_unique<ColDescriptor>("sampleIds", DataType(DataType::DE_INT64), TensorImpl::kFlexible, 1);
}
TensorShape shape(std::vector<dsize_t>(1, num_elements));
RETURN_IF_NOT_OK(Tensor::CreateTensor(sample_ids, col_desc_->tensorImpl(), shape, col_desc_->type()));
RETURN_IF_NOT_OK(
(*sample_ids)->AllocateBuffer((*sample_ids)->SizeInBytes())); // allocate memory in case user forgets!
RETURN_IF_NOT_OK(Tensor::CreateEmpty(shape, col_desc_->type(), sample_ids));
return Status::OK();
}

Some files were not shown because too many files have changed in this diff Show More

Loading…
Cancel
Save