diff --git a/mindspore/ccsrc/minddata/dataset/kernels/tensor_op.cc b/mindspore/ccsrc/minddata/dataset/kernels/tensor_op.cc index c22504fe00..d1c76ab424 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/tensor_op.cc +++ b/mindspore/ccsrc/minddata/dataset/kernels/tensor_op.cc @@ -40,8 +40,7 @@ Status TensorOp::Compute(const std::shared_ptr &input, std::shared_ptrresize(1); return Compute(input[0], &(*output)[0]); } diff --git a/mindspore/dataset/text/transforms.py b/mindspore/dataset/text/transforms.py index 9d1120a0ec..7a883e01e0 100644 --- a/mindspore/dataset/text/transforms.py +++ b/mindspore/dataset/text/transforms.py @@ -63,24 +63,6 @@ class TextTensorOperation(TensorOperation): """ Base class of Text Tensor Ops """ - def __call__(self, *tensor_list): - tensor_array = [] - output_list = [] - # Combine input tensor_list to a TensorRow - for input_tensor in tensor_list: - if not isinstance(input_tensor, (str, list)): - raise TypeError("Input should be string or list of strings, got {}.".format(type(input_tensor))) - tensor_array.append(cde.Tensor(np.asarray(input_tensor))) - callable_op = cde.Execute(self.parse()) - output_list = callable_op(tensor_array) - for i, element in enumerate(output_list): - arr = element.as_array() - if arr.dtype.char == 'S': - output_list[i] = np.char.decode(arr) - else: - output_list[i] = arr - return output_list[0] if len(output_list) == 1 else output_list - def parse(self): raise NotImplementedError("TextTensorOperation has to implement parse() method.") diff --git a/mindspore/dataset/transforms/c_transforms.py b/mindspore/dataset/transforms/c_transforms.py index 930bc37104..39005d5c40 100644 --- a/mindspore/dataset/transforms/c_transforms.py +++ b/mindspore/dataset/transforms/c_transforms.py @@ -27,8 +27,20 @@ from ..core.datatypes import mstype_to_detype class TensorOperation: - def __call__(self): - raise NotImplementedError("TensorOperation has to implement __call__() method.") + """ + Base class Tensor Ops + """ + def __call__(self, *input_tensor_list): + tensor_row = [cde.Tensor(np.asarray(tensor)) for tensor in input_tensor_list] + callable_op = cde.Execute(self.parse()) + output_tensor_list = callable_op(tensor_row) + for i, element in enumerate(output_tensor_list): + arr = element.as_array() + if arr.dtype.char == 'S': + output_tensor_list[i] = np.char.decode(arr) + else: + output_tensor_list[i] = arr + return output_tensor_list[0] if len(output_tensor_list) == 1 else tuple(output_tensor_list) def parse(self): raise NotImplementedError("TensorOperation has to implement parse() method.") diff --git a/mindspore/dataset/vision/c_transforms.py b/mindspore/dataset/vision/c_transforms.py index 7fb7cfc4df..ade2c41228 100644 --- a/mindspore/dataset/vision/c_transforms.py +++ b/mindspore/dataset/vision/c_transforms.py @@ -62,24 +62,11 @@ class ImageTensorOperation(TensorOperation): """ Base class of Image Tensor Ops """ - def __call__(self, *tensor_list): - tensor_array = [] - output_list = [] - # Combine input tensor_list to a TensorRow - for input_tensor in tensor_list: - if not isinstance(input_tensor, (np.ndarray, Image.Image)): - raise TypeError("Input should be NumPy or PIL image, got {}.".format(type(input_tensor))) - tensor_array.append(cde.Tensor(np.asarray(input_tensor))) - callable_op = cde.Execute(self.parse()) - output_list = callable_op(tensor_array) - - for i, element in enumerate(output_list): - arr = element.as_array() - if arr.dtype.char == 'S': - output_list[i] = np.char.decode(arr) - else: - output_list[i] = arr - return output_list[0] if len(output_list) == 1 else output_list + def __call__(self, *input_tensor_list): + for tensor in input_tensor_list: + if not isinstance(tensor, (np.ndarray, Image.Image)): + raise TypeError("Input should be NumPy or PIL image, got {}.".format(type(tensor))) + return super().__call__(*input_tensor_list) def parse(self): raise NotImplementedError("ImageTensorOperation has to implement parse() method.") @@ -285,9 +272,7 @@ class Decode(ImageTensorOperation): """ if not isinstance(img, np.ndarray) or img.ndim != 1 or img.dtype.type is np.str_: raise TypeError("Input should be an encoded image with 1-D NumPy type, got {}.".format(type(img))) - decode = cde.Execute(cde.DecodeOperation(self.rgb)) - img = decode(cde.Tensor(np.asarray(img))) - return img.as_array() + return super().__call__(img) def parse(self): return cde.DecodeOperation(self.rgb) diff --git a/tests/ut/python/dataset/test_HWC2CHW.py b/tests/ut/python/dataset/test_HWC2CHW.py index 75f372466b..a9186212b7 100644 --- a/tests/ut/python/dataset/test_HWC2CHW.py +++ b/tests/ut/python/dataset/test_HWC2CHW.py @@ -35,14 +35,12 @@ def test_HWC2CHW_callable(): Test HWC2CHW is callable """ logger.info("Test HWC2CHW callable") - img = np.fromfile("../data/dataset/apple.jpg", dtype=np.uint8) - logger.info("Image.type: {}, Image.shape: {}".format(type(img), img.shape)) - img = c_vision.Decode()(img) - assert img.shape == (2268, 4032, 3) + img = np.zeros([50, 50, 3]) + assert img.shape == (50, 50, 3) # test one tensor img1 = c_vision.HWC2CHW()(img) - assert img1.shape == (3, 2268, 4032) + assert img1.shape == (3, 50, 50) # test input multiple tensors with pytest.raises(RuntimeError) as info: @@ -55,7 +53,6 @@ def test_HWC2CHW_callable(): assert "The op is OneToOne, can only accept one tensor as input." in str(info.value) - def test_HWC2CHW(plot=False): """ Test HWC2CHW diff --git a/tests/ut/python/dataset/test_ngram_op.py b/tests/ut/python/dataset/test_ngram_op.py index ec378b4455..f1ce050520 100644 --- a/tests/ut/python/dataset/test_ngram_op.py +++ b/tests/ut/python/dataset/test_ngram_op.py @@ -20,6 +20,24 @@ import mindspore.dataset as ds import mindspore.dataset.text as text +def test_ngram_callable(): + """ + Test ngram op is callable + """ + op = text.Ngram(2, separator="-") + + input1 = " WildRose Country" + input1 = np.array(input1.split(" "), dtype='S') + expect1 = ['-WildRose', 'WildRose-Country'] + result1 = op(input1) + assert np.array_equal(result1, expect1) + + input2 = ["WildRose Country", "Canada's Ocean Playground", "Land of Living Skies"] + expect2 = ["WildRose Country-Canada's Ocean Playground", "Canada's Ocean Playground-Land of Living Skies"] + result2 = op(input2) + assert np.array_equal(result2, expect2) + + def test_multiple_ngrams(): """ test n-gram where n is a list of integers""" plates_mottos = ["WildRose Country", "Canada's Ocean Playground", "Land of Living Skies"] @@ -105,6 +123,7 @@ def test_corner_cases(): if __name__ == '__main__': + test_ngram_callable() test_multiple_ngrams() test_simple_ngram() test_corner_cases() diff --git a/tests/ut/python/dataset/test_pair_truncate.py b/tests/ut/python/dataset/test_pair_truncate.py index da35ee1dd7..ae3225212c 100644 --- a/tests/ut/python/dataset/test_pair_truncate.py +++ b/tests/ut/python/dataset/test_pair_truncate.py @@ -30,6 +30,17 @@ def compare(in1, in2, length, out1, out2): np.testing.assert_array_equal(out2, d["s2"]) +def test_callable(): + op = text.TruncateSequencePair(3) + data = [["1", "2", "3"], ["4", "5"]] + result_text = op(*data) + column1, column2 = op(["1", "2", "3"], ["4", "5"]) + assert np.array_equal(result_text[0], ['1', '2']) + assert np.array_equal(result_text[1], ['4']) + assert np.array_equal(column1, ['1', '2']) + assert np.array_equal(column2, ['4']) + + def test_basics(): compare(in1=[1, 2, 3], in2=[4, 5], length=4, out1=[1, 2], out2=[4, 5]) compare(in1=[1, 2], in2=[4, 5], length=4, out1=[1, 2], out2=[4, 5]) @@ -59,6 +70,7 @@ def test_exceptions(): if __name__ == "__main__": + test_callable() test_basics() test_basics_odd() test_basics_str() diff --git a/tests/ut/python/dataset/test_sliding_window.py b/tests/ut/python/dataset/test_sliding_window.py index ab14f1f672..eb77944f8d 100644 --- a/tests/ut/python/dataset/test_sliding_window.py +++ b/tests/ut/python/dataset/test_sliding_window.py @@ -16,10 +16,34 @@ Testing SlidingWindow in mindspore.dataset """ import numpy as np +import pytest import mindspore.dataset as ds import mindspore.dataset.text as text +def test_sliding_window_callable(): + """ + Test sliding window op is callable + """ + op = text.SlidingWindow(2, 0) + + input1 = ["大", "家", "早", "上", "好"] + expect = np.array([['大', '家'], ['家', '早'], ['早', '上'], ['上', '好']]) + result = op(input1) + assert np.array_equal(result, expect) + + # test 2D input + input2 = [["大", "家", "早", "上", "好"]] + with pytest.raises(RuntimeError) as info: + _ = op(input2) + assert "SlidingWindow: SlidingWindow supports 1D input only for now." in str(info.value) + + # test input multiple tensors + with pytest.raises(RuntimeError) as info: + _ = op(input1, input1) + assert "The op is OneToOne, can only accept one tensor as input." in str(info.value) + + def test_sliding_window_string(): """ test sliding_window with string type""" inputs = [["大", "家", "早", "上", "好"]] @@ -104,6 +128,7 @@ def test_sliding_window_exception(): if __name__ == '__main__': + test_sliding_window_callable() test_sliding_window_string() test_sliding_window_number() test_sliding_window_big_width() diff --git a/tests/ut/python/dataset/test_to_number_op.py b/tests/ut/python/dataset/test_to_number_op.py index 4eaba99b4f..4fa37e90e1 100644 --- a/tests/ut/python/dataset/test_to_number_op.py +++ b/tests/ut/python/dataset/test_to_number_op.py @@ -50,6 +50,12 @@ def test_to_number_eager(): _ = op(*input_strings) assert "The op is OneToOne, can only accept one tensor as input." in str(info.value) + # test input invalid tensor + invalid_input = [["1", "2", "3"], ["4", "5"]] + with pytest.raises(RuntimeError) as info: + _ = op(invalid_input) + assert "Invalid data type." in str(info.value) + def test_to_number_typical_case_integral(): input_strings = [["-121", "14"], ["-2219", "7623"], ["-8162536", "162371864"],