mindspore-ci-bot
5 years ago
committed by
Gitee
commit
ec8f541325
@ -0,0 +1,241 @@
|
||||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "dataset/text/kernels/to_number_op.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <limits>
|
||||
#include <memory>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "dataset/core/data_type.h"
|
||||
#include "dataset/core/tensor.h"
|
||||
#include "dataset/core/tensor_shape.h"
|
||||
#include "dataset/kernels/data/data_utils.h"
|
||||
#include "dataset/util/status.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace dataset {
|
||||
|
||||
ToNumberOp::ToNumberOp(const DataType &cast_to_type) : cast_to_type_(cast_to_type) {}
|
||||
|
||||
ToNumberOp::ToNumberOp(const std::string &cast_to_type) : cast_to_type_(DataType(cast_to_type)) {}
|
||||
|
||||
Status ToNumberOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(input->type() == DataType::DE_STRING, "Input tenosrs should have type string.");
|
||||
|
||||
switch (cast_to_type_.value()) {
|
||||
case DataType::DE_INT8:
|
||||
RETURN_IF_NOT_OK(ToSignedIntegral<int8_t>(input, output));
|
||||
break;
|
||||
case DataType::DE_INT16:
|
||||
RETURN_IF_NOT_OK(ToSignedIntegral<int16_t>(input, output));
|
||||
break;
|
||||
case DataType::DE_INT32:
|
||||
RETURN_IF_NOT_OK(ToSignedIntegral<int32_t>(input, output));
|
||||
break;
|
||||
case DataType::DE_INT64:
|
||||
RETURN_IF_NOT_OK(ToSignedIntegral<int64_t>(input, output));
|
||||
break;
|
||||
case DataType::DE_UINT8:
|
||||
RETURN_IF_NOT_OK(ToUnsignedIntegral<uint8_t>(input, output));
|
||||
break;
|
||||
case DataType::DE_UINT16:
|
||||
RETURN_IF_NOT_OK(ToUnsignedIntegral<uint16_t>(input, output));
|
||||
break;
|
||||
case DataType::DE_UINT32:
|
||||
RETURN_IF_NOT_OK(ToUnsignedIntegral<uint32_t>(input, output));
|
||||
break;
|
||||
case DataType::DE_UINT64:
|
||||
RETURN_IF_NOT_OK(ToUnsignedIntegral<uint64_t>(input, output));
|
||||
break;
|
||||
case DataType::DE_FLOAT16:
|
||||
RETURN_IF_NOT_OK(this->ToFloat16(input, output));
|
||||
break;
|
||||
case DataType::DE_FLOAT32:
|
||||
RETURN_IF_NOT_OK(ToFloat(input, output));
|
||||
break;
|
||||
case DataType::DE_FLOAT64:
|
||||
RETURN_IF_NOT_OK(ToDouble(input, output));
|
||||
break;
|
||||
}
|
||||
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
void ToNumberOp::Print(std::ostream &out) const { out << "ToNumberOp: casting to " << '\n'; }
|
||||
|
||||
Status ToNumberOp::OutputShape(const std::vector<TensorShape> &input_shapes, std::vector<TensorShape> &output_shapes) {
|
||||
(void)std::copy(input_shapes.begin(), input_shapes.end(), std::back_inserter(output_shapes));
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
Status ToNumberOp::ToSignedIntegral(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
|
||||
std::vector<T> casted;
|
||||
|
||||
for (auto it = input->begin<std::string_view>(); it != input->end<std::string_view>(); ++it) {
|
||||
bool is_cast_out_of_range = false;
|
||||
int64_t result = 0;
|
||||
|
||||
try {
|
||||
result = std::stoll(std::string(*it));
|
||||
} catch (const std::out_of_range &) {
|
||||
is_cast_out_of_range = true;
|
||||
} catch (const std::invalid_argument &) {
|
||||
RETURN_STATUS_UNEXPECTED("It is invalid to convert " + std::string(*it) + " to a number.");
|
||||
}
|
||||
|
||||
if (result > std::numeric_limits<T>::max() || result < std::numeric_limits<T>::min() || is_cast_out_of_range) {
|
||||
std::string error_message = "String input " + std::string(*it) + " will be out of bounds if casted to " +
|
||||
cast_to_type_.ToString() + ". The valid range is: [" +
|
||||
std::to_string(std::numeric_limits<T>::min()) + ", " +
|
||||
std::to_string(std::numeric_limits<T>::max()) + "].";
|
||||
|
||||
RETURN_STATUS_UNEXPECTED(error_message);
|
||||
}
|
||||
|
||||
T casted_result = static_cast<T>(result);
|
||||
casted.push_back(casted_result);
|
||||
}
|
||||
|
||||
RETURN_IF_NOT_OK(Tensor::CreateTensor(output, casted, input->shape()));
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
Status ToNumberOp::ToUnsignedIntegral(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
|
||||
std::vector<T> casted;
|
||||
|
||||
for (auto it = input->begin<std::string_view>(); it != input->end<std::string_view>(); ++it) {
|
||||
bool is_cast_out_of_range = false;
|
||||
uint64_t result = 0;
|
||||
|
||||
// If there is a - at the start of the string, it is considered by us to
|
||||
// be out of bounds. If the - is somewhere else in the string, it is
|
||||
// deemed invalid by std::stoull and will throw std::invalid_argument
|
||||
for (int i = 0; i < (*it).size(); i++) {
|
||||
if ((*it)[i] == '-') {
|
||||
is_cast_out_of_range = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
result = std::stoull(std::string(*it));
|
||||
} catch (const std::out_of_range &) {
|
||||
is_cast_out_of_range = true;
|
||||
} catch (const std::invalid_argument &) {
|
||||
RETURN_STATUS_UNEXPECTED("It is invalid to convert " + std::string(*it) + " to an unsigned integer.");
|
||||
}
|
||||
|
||||
if (result > std::numeric_limits<T>::max() || result < std::numeric_limits<T>::min() || is_cast_out_of_range) {
|
||||
std::string error_message = "String input " + std::string(*it) + " will be out of bounds if casted to " +
|
||||
cast_to_type_.ToString() + ". The valid range is: [" +
|
||||
std::to_string(std::numeric_limits<T>::min()) + ", " +
|
||||
std::to_string(std::numeric_limits<T>::max()) + "].";
|
||||
|
||||
RETURN_STATUS_UNEXPECTED(error_message);
|
||||
}
|
||||
|
||||
T casted_result = static_cast<T>(result);
|
||||
casted.push_back(casted_result);
|
||||
}
|
||||
|
||||
RETURN_IF_NOT_OK(Tensor::CreateTensor(output, casted, input->shape()));
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status ToNumberOp::ToFloat16(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
|
||||
// special case, float16 does not exist in c++, no native support for
|
||||
// casting, so cast to float first then use this method, which use Eigen.
|
||||
std::shared_ptr<Tensor> temp;
|
||||
RETURN_IF_NOT_OK(Tensor::CreateTensor(&temp, TensorImpl::kFlexible, input->shape(), DataType("float32")));
|
||||
RETURN_IF_NOT_OK(ToFloat(input, &temp));
|
||||
RETURN_IF_NOT_OK(mindspore::dataset::ToFloat16(temp, output));
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status ToNumberOp::ToFloat(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
|
||||
std::vector<float> casted;
|
||||
|
||||
for (auto it = input->begin<std::string_view>(); it != input->end<std::string_view>(); ++it) {
|
||||
bool is_cast_out_of_range = false;
|
||||
float result = 0;
|
||||
|
||||
try {
|
||||
result = std::stof(std::string(*it));
|
||||
} catch (const std::out_of_range &) {
|
||||
is_cast_out_of_range = true;
|
||||
} catch (const std::invalid_argument &) {
|
||||
RETURN_STATUS_UNEXPECTED("It is invalid to convert " + std::string(*it) + " to an unsigned integer.");
|
||||
}
|
||||
|
||||
if (result > std::numeric_limits<float>::max() || result < std::numeric_limits<float>::lowest() ||
|
||||
is_cast_out_of_range) {
|
||||
std::string error_message = "String input " + std::string(*it) + " will be out of bounds if casted to " +
|
||||
cast_to_type_.ToString() + ". The valid range is: [" +
|
||||
std::to_string(std::numeric_limits<float>::lowest()) + ", " +
|
||||
std::to_string(std::numeric_limits<float>::max()) + "].";
|
||||
|
||||
RETURN_STATUS_UNEXPECTED(error_message);
|
||||
}
|
||||
|
||||
float casted_result = static_cast<float>(result);
|
||||
casted.push_back(casted_result);
|
||||
}
|
||||
|
||||
RETURN_IF_NOT_OK(Tensor::CreateTensor(output, casted, input->shape()));
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status ToNumberOp::ToDouble(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
|
||||
std::vector<double> casted;
|
||||
|
||||
for (auto it = input->begin<std::string_view>(); it != input->end<std::string_view>(); ++it) {
|
||||
bool is_cast_out_of_range = false;
|
||||
double result = 0;
|
||||
|
||||
try {
|
||||
result = std::stod(std::string(*it));
|
||||
} catch (const std::out_of_range &) {
|
||||
is_cast_out_of_range = true;
|
||||
} catch (const std::invalid_argument &) {
|
||||
RETURN_STATUS_UNEXPECTED("It is invalid to convert " + std::string(*it) + " to an unsigned integer.");
|
||||
}
|
||||
|
||||
if (result > std::numeric_limits<double>::max() || result < std::numeric_limits<double>::lowest() ||
|
||||
is_cast_out_of_range) {
|
||||
std::string error_message = "String input " + std::string(*it) + " will be out of bounds if casted to " +
|
||||
cast_to_type_.ToString() + ". The valid range is: [" +
|
||||
std::to_string(std::numeric_limits<double>::lowest()) + ", " +
|
||||
std::to_string(std::numeric_limits<double>::max()) + "].";
|
||||
|
||||
RETURN_STATUS_UNEXPECTED(error_message);
|
||||
}
|
||||
|
||||
double casted_result = static_cast<double>(result);
|
||||
casted.push_back(casted_result);
|
||||
}
|
||||
|
||||
RETURN_IF_NOT_OK(Tensor::CreateTensor(output, casted, input->shape()));
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
} // namespace dataset
|
||||
} // namespace mindspore
|
||||
@ -0,0 +1,79 @@
|
||||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef DATASET_TEXT_KERNELS_TO_NUMBER_OP_H_
|
||||
#define DATASET_TEXT_KERNELS_TO_NUMBER_OP_H_
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "dataset/core/data_type.h"
|
||||
#include "dataset/core/tensor.h"
|
||||
#include "dataset/kernels/tensor_op.h"
|
||||
#include "dataset/util/status.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace dataset {
|
||||
|
||||
class ToNumberOp : public TensorOp {
|
||||
public:
|
||||
// Constructor of ToNumberOp
|
||||
// @param const DataType &cast_to_type - the type to convert string inputs to.
|
||||
explicit ToNumberOp(const DataType &cast_to_type);
|
||||
|
||||
// Constructor of ToNumberOp
|
||||
// @param const std::string &cast_to_type - the type in string form to convert string inputs to.
|
||||
explicit ToNumberOp(const std::string &cast_to_type);
|
||||
|
||||
~ToNumberOp() override = default;
|
||||
|
||||
// Perform numeric conversion on each string in each tensor.
|
||||
// @param const std::shared_ptr<Tensor> &input
|
||||
// @param std::shared_ptr<Tensor> *output
|
||||
// @return error code
|
||||
Status Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) override;
|
||||
|
||||
// For each input shape, find the output shape
|
||||
// @param std::vector<TensorShape> &inputs - shape of input tensors
|
||||
// @param std::vector<TensorShape> &outputs - shape of output tensors
|
||||
// @return error code
|
||||
Status OutputShape(const std::vector<TensorShape> &input_shapes, std::vector<TensorShape> &output_shapes) override;
|
||||
|
||||
// print arg for debugging
|
||||
// @param std::ostream &out
|
||||
void Print(std::ostream &out) const override;
|
||||
|
||||
private:
|
||||
template <typename T>
|
||||
Status ToSignedIntegral(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output);
|
||||
|
||||
template <typename T>
|
||||
Status ToUnsignedIntegral(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output);
|
||||
|
||||
Status ToFloat16(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output);
|
||||
|
||||
Status ToFloat(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output);
|
||||
|
||||
Status ToDouble(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output);
|
||||
|
||||
DataType cast_to_type_;
|
||||
};
|
||||
|
||||
} // namespace dataset
|
||||
} // namespace mindspore
|
||||
|
||||
#endif // DATASET_TEXT_KERNELS_TO_NUMBER_OP_H_
|
||||
@ -0,0 +1,194 @@
|
||||
# Copyright 2020 Huawei Technologies Co., Ltd
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
import mindspore.common.dtype as mstype
|
||||
import mindspore.dataset as ds
|
||||
import mindspore.dataset.text as text
|
||||
|
||||
np_integral_types = [np.int8, np.int16, np.int32, np.int64, np.uint8, np.uint16,
|
||||
np.uint32, np.uint64]
|
||||
ms_integral_types = [mstype.int8, mstype.int16, mstype.int32, mstype.int64, mstype.uint8,
|
||||
mstype.uint16, mstype.uint32, mstype.uint64]
|
||||
|
||||
np_non_integral_types = [np.float16, np.float32, np.float64]
|
||||
ms_non_integral_types = [mstype.float16, mstype.float32, mstype.float64]
|
||||
|
||||
def string_dataset_generator(strings):
|
||||
for string in strings:
|
||||
yield (np.array(string, dtype='S'),)
|
||||
|
||||
|
||||
def test_to_number_typical_case_integral():
|
||||
input_strings = [["-121", "14"], ["-2219", "7623"], ["-8162536", "162371864"],
|
||||
["-1726483716", "98921728421"]]
|
||||
|
||||
for ms_type, inputs in zip(ms_integral_types, input_strings):
|
||||
dataset = ds.GeneratorDataset(string_dataset_generator(inputs), "strings")
|
||||
dataset = dataset.map(input_columns=["strings"], operations=text.ToNumber(ms_type))
|
||||
|
||||
expected_output = [int(string) for string in inputs]
|
||||
output = []
|
||||
for data in dataset.create_dict_iterator():
|
||||
output.append(data["strings"])
|
||||
|
||||
assert output == expected_output
|
||||
|
||||
|
||||
def test_to_number_typical_case_non_integral():
|
||||
input_strings = [["-1.1", "1.4"], ["-2219.321", "7623.453"], ["-816256.234282", "162371864.243243"]]
|
||||
epsilons = [0.001, 0.001, 0.0001, 0.0001, 0.0000001, 0.0000001]
|
||||
|
||||
for ms_type, inputs in zip(ms_non_integral_types, input_strings):
|
||||
dataset = ds.GeneratorDataset(string_dataset_generator(inputs), "strings")
|
||||
dataset = dataset.map(input_columns=["strings"], operations=text.ToNumber(ms_type))
|
||||
|
||||
expected_output = [float(string) for string in inputs]
|
||||
output = []
|
||||
for data in dataset.create_dict_iterator():
|
||||
output.append(data["strings"])
|
||||
|
||||
for expected, actual, epsilon in zip(expected_output, output, epsilons):
|
||||
assert abs(expected - actual) < epsilon
|
||||
|
||||
|
||||
def out_of_bounds_error_message_check(dataset, np_type, value_to_cast):
|
||||
type_info = np.iinfo(np_type)
|
||||
type_max = str(type_info.max)
|
||||
type_min = str(type_info.min)
|
||||
type_name = str(np.dtype(np_type))
|
||||
|
||||
with pytest.raises(RuntimeError) as info:
|
||||
for _ in dataset.create_dict_iterator():
|
||||
pass
|
||||
assert "String input " + value_to_cast + " will be out of bounds if casted to " + type_name in str(info.value)
|
||||
assert "valid range is: [" + type_min + ", " + type_max + "]" in str(info.value)
|
||||
|
||||
|
||||
def test_to_number_out_of_bounds_integral():
|
||||
for np_type, ms_type in zip(np_integral_types, ms_integral_types):
|
||||
type_info = np.iinfo(np_type)
|
||||
input_strings = [str(type_info.max + 10)]
|
||||
dataset = ds.GeneratorDataset(string_dataset_generator(input_strings), "strings")
|
||||
dataset = dataset.map(input_columns=["strings"], operations=text.ToNumber(ms_type))
|
||||
out_of_bounds_error_message_check(dataset, np_type, input_strings[0])
|
||||
|
||||
input_strings = [str(type_info.min - 10)]
|
||||
dataset = ds.GeneratorDataset(string_dataset_generator(input_strings), "strings")
|
||||
dataset = dataset.map(input_columns=["strings"], operations=text.ToNumber(ms_type))
|
||||
out_of_bounds_error_message_check(dataset, np_type, input_strings[0])
|
||||
|
||||
|
||||
def test_to_number_out_of_bounds_non_integral():
|
||||
above_range = [str(np.finfo(np.float16).max * 10), str(np.finfo(np.float32).max * 10), "1.8e+308"]
|
||||
|
||||
input_strings = [above_range[0]]
|
||||
dataset = ds.GeneratorDataset(string_dataset_generator(input_strings), "strings")
|
||||
dataset = dataset.map(input_columns=["strings"], operations=text.ToNumber(ms_non_integral_types[0]))
|
||||
|
||||
with pytest.raises(RuntimeError) as info:
|
||||
for _ in dataset.create_dict_iterator():
|
||||
pass
|
||||
assert "outside of valid float16 range" in str(info.value)
|
||||
|
||||
input_strings = [above_range[1]]
|
||||
dataset = ds.GeneratorDataset(string_dataset_generator(input_strings), "strings")
|
||||
dataset = dataset.map(input_columns=["strings"], operations=text.ToNumber(ms_non_integral_types[1]))
|
||||
|
||||
with pytest.raises(RuntimeError) as info:
|
||||
for _ in dataset.create_dict_iterator():
|
||||
pass
|
||||
assert "String input " + input_strings[0] + " will be out of bounds if casted to float32" in str(info.value)
|
||||
|
||||
input_strings = [above_range[2]]
|
||||
dataset = ds.GeneratorDataset(string_dataset_generator(input_strings), "strings")
|
||||
dataset = dataset.map(input_columns=["strings"], operations=text.ToNumber(ms_non_integral_types[2]))
|
||||
|
||||
with pytest.raises(RuntimeError) as info:
|
||||
for _ in dataset.create_dict_iterator():
|
||||
pass
|
||||
assert "String input " + input_strings[0] + " will be out of bounds if casted to float64" in str(info.value)
|
||||
|
||||
below_range = [str(np.finfo(np.float16).min * 10), str(np.finfo(np.float32).min * 10), "-1.8e+308"]
|
||||
|
||||
input_strings = [below_range[0]]
|
||||
dataset = ds.GeneratorDataset(string_dataset_generator(input_strings), "strings")
|
||||
dataset = dataset.map(input_columns=["strings"], operations=text.ToNumber(ms_non_integral_types[0]))
|
||||
|
||||
with pytest.raises(RuntimeError) as info:
|
||||
for _ in dataset.create_dict_iterator():
|
||||
pass
|
||||
assert "outside of valid float16 range" in str(info.value)
|
||||
|
||||
input_strings = [below_range[1]]
|
||||
dataset = ds.GeneratorDataset(string_dataset_generator(input_strings), "strings")
|
||||
dataset = dataset.map(input_columns=["strings"], operations=text.ToNumber(ms_non_integral_types[1]))
|
||||
|
||||
with pytest.raises(RuntimeError) as info:
|
||||
for _ in dataset.create_dict_iterator():
|
||||
pass
|
||||
assert "String input " + input_strings[0] + " will be out of bounds if casted to float32" in str(info.value)
|
||||
|
||||
input_strings = [below_range[2]]
|
||||
dataset = ds.GeneratorDataset(string_dataset_generator(input_strings), "strings")
|
||||
dataset = dataset.map(input_columns=["strings"], operations=text.ToNumber(ms_non_integral_types[2]))
|
||||
|
||||
with pytest.raises(RuntimeError) as info:
|
||||
for _ in dataset.create_dict_iterator():
|
||||
pass
|
||||
assert "String input " + input_strings[0] + " will be out of bounds if casted to float64" in str(info.value)
|
||||
|
||||
|
||||
def test_to_number_boundaries_integral():
|
||||
for np_type, ms_type in zip(np_integral_types, ms_integral_types):
|
||||
type_info = np.iinfo(np_type)
|
||||
input_strings = [str(type_info.max)]
|
||||
dataset = ds.GeneratorDataset(string_dataset_generator(input_strings), "strings")
|
||||
dataset = dataset.map(input_columns=["strings"], operations=text.ToNumber(ms_type))
|
||||
for data in dataset.create_dict_iterator():
|
||||
assert data["strings"] == int(input_strings[0])
|
||||
|
||||
input_strings = [str(type_info.min)]
|
||||
dataset = ds.GeneratorDataset(string_dataset_generator(input_strings), "strings")
|
||||
dataset = dataset.map(input_columns=["strings"], operations=text.ToNumber(ms_type))
|
||||
for data in dataset.create_dict_iterator():
|
||||
assert data["strings"] == int(input_strings[0])
|
||||
|
||||
input_strings = [str(0)]
|
||||
dataset = ds.GeneratorDataset(string_dataset_generator(input_strings), "strings")
|
||||
dataset = dataset.map(input_columns=["strings"], operations=text.ToNumber(ms_type))
|
||||
for data in dataset.create_dict_iterator():
|
||||
assert data["strings"] == int(input_strings[0])
|
||||
|
||||
|
||||
def test_to_number_invalid_input():
|
||||
input_strings = ["a8fa9ds8fa"]
|
||||
dataset = ds.GeneratorDataset(string_dataset_generator(input_strings), "strings")
|
||||
dataset = dataset.map(input_columns=["strings"], operations=text.ToNumber(mstype.int32))
|
||||
|
||||
with pytest.raises(RuntimeError) as info:
|
||||
for _ in dataset.create_dict_iterator():
|
||||
pass
|
||||
assert "It is invalid to convert " + input_strings[0] + " to a number" in str(info.value)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
test_to_number_typical_case_integral()
|
||||
test_to_number_typical_case_non_integral()
|
||||
test_to_number_boundaries_integral()
|
||||
test_to_number_out_of_bounds_integral()
|
||||
test_to_number_out_of_bounds_non_integral()
|
||||
test_to_number_invalid_input()
|
||||
Loading…
Reference in new issue