!12843 Add more callable test and clean the input check for tensor ops

From: @alexyuyue
Reviewed-by: @robingrosman,@nsyca
Signed-off-by: @robingrosman

mindspore/ccsrc/minddata/dataset/kernels/tensor_op.cc

@@ -40,8 +40,7 @@ Status TensorOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<T
 Status TensorOp::Compute(const TensorRow &input, TensorRow *output) {
   IO_CHECK_VECTOR(input, output);
   if (OneToOne()) {
-    if (input.size() != 1)
-      return Status(StatusCode::kMDUnexpectedError, "The op is OneToOne, can only accept one tensor as input.");
+    CHECK_FAIL_RETURN_UNEXPECTED(input.size() == 1, "The op is OneToOne, can only accept one tensor as input.");
     output->resize(1);
     return Compute(input[0], &(*output)[0]);
   }

mindspore/dataset/text/transforms.py

@@ -63,24 +63,6 @@ class TextTensorOperation(TensorOperation):
     """
     Base class of Text Tensor Ops
     """
-    def __call__(self, *tensor_list):
-        tensor_array = []
-        output_list = []
-        # Combine input tensor_list to a TensorRow
-        for input_tensor in tensor_list:
-            if not isinstance(input_tensor, (str, list)):
-                raise TypeError("Input should be string or list of strings, got {}.".format(type(input_tensor)))
-            tensor_array.append(cde.Tensor(np.asarray(input_tensor)))
-        callable_op = cde.Execute(self.parse())
-        output_list = callable_op(tensor_array)
-        for i, element in enumerate(output_list):
-            arr = element.as_array()
-            if arr.dtype.char == 'S':
-                output_list[i] = np.char.decode(arr)
-            else:
-                output_list[i] = arr
-        return output_list[0] if len(output_list) == 1 else output_list
-
     def parse(self):
         raise NotImplementedError("TextTensorOperation has to implement parse() method.")
 

mindspore/dataset/transforms/c_transforms.py

@@ -27,8 +27,20 @@ from ..core.datatypes import mstype_to_detype
 
 
 class TensorOperation:
-    def __call__(self):
-        raise NotImplementedError("TensorOperation has to implement __call__() method.")
+    """
+    Base class Tensor Ops
+    """
+    def __call__(self, *input_tensor_list):
+        tensor_row = [cde.Tensor(np.asarray(tensor)) for tensor in input_tensor_list]
+        callable_op = cde.Execute(self.parse())
+        output_tensor_list = callable_op(tensor_row)
+        for i, element in enumerate(output_tensor_list):
+            arr = element.as_array()
+            if arr.dtype.char == 'S':
+                output_tensor_list[i] = np.char.decode(arr)
+            else:
+                output_tensor_list[i] = arr
+        return output_tensor_list[0] if len(output_tensor_list) == 1 else tuple(output_tensor_list)
 
     def parse(self):
         raise NotImplementedError("TensorOperation has to implement parse() method.")

mindspore/dataset/vision/c_transforms.py

@@ -62,24 +62,11 @@ class ImageTensorOperation(TensorOperation):
     """
     Base class of Image Tensor Ops
     """
-    def __call__(self, *tensor_list):
-        tensor_array = []
-        output_list = []
-        # Combine input tensor_list to a TensorRow
-        for input_tensor in tensor_list:
-            if not isinstance(input_tensor, (np.ndarray, Image.Image)):
-                raise TypeError("Input should be NumPy or PIL image, got {}.".format(type(input_tensor)))
-            tensor_array.append(cde.Tensor(np.asarray(input_tensor)))
-        callable_op = cde.Execute(self.parse())
-        output_list = callable_op(tensor_array)
-
-        for i, element in enumerate(output_list):
-            arr = element.as_array()
-            if arr.dtype.char == 'S':
-                output_list[i] = np.char.decode(arr)
-            else:
-                output_list[i] = arr
-        return output_list[0] if len(output_list) == 1 else output_list
+    def __call__(self, *input_tensor_list):
+        for tensor in input_tensor_list:
+            if not isinstance(tensor, (np.ndarray, Image.Image)):
+                raise TypeError("Input should be NumPy or PIL image, got {}.".format(type(tensor)))
+        return super().__call__(*input_tensor_list)
 
     def parse(self):
         raise NotImplementedError("ImageTensorOperation has to implement parse() method.")
@@ -285,9 +272,7 @@ class Decode(ImageTensorOperation):
         """
         if not isinstance(img, np.ndarray) or img.ndim != 1 or img.dtype.type is np.str_:
             raise TypeError("Input should be an encoded image with 1-D NumPy type, got {}.".format(type(img)))
-        decode = cde.Execute(cde.DecodeOperation(self.rgb))
-        img = decode(cde.Tensor(np.asarray(img)))
-        return img.as_array()
+        return super().__call__(img)
 
     def parse(self):
         return cde.DecodeOperation(self.rgb)

tests/ut/python/dataset/test_HWC2CHW.py

@@ -35,14 +35,12 @@ def test_HWC2CHW_callable():
     Test HWC2CHW is callable
     """
     logger.info("Test HWC2CHW callable")
-    img = np.fromfile("../data/dataset/apple.jpg", dtype=np.uint8)
-    logger.info("Image.type: {}, Image.shape: {}".format(type(img), img.shape))
-    img = c_vision.Decode()(img)
-    assert img.shape == (2268, 4032, 3)
+    img = np.zeros([50, 50, 3])
+    assert img.shape == (50, 50, 3)
 
     # test one tensor
     img1 = c_vision.HWC2CHW()(img)
-    assert img1.shape == (3, 2268, 4032)
+    assert img1.shape == (3, 50, 50)
 
     # test input multiple tensors
     with pytest.raises(RuntimeError) as info:
@@ -55,7 +53,6 @@ def test_HWC2CHW_callable():
     assert "The op is OneToOne, can only accept one tensor as input." in str(info.value)
 
 
-
 def test_HWC2CHW(plot=False):
     """
     Test HWC2CHW

tests/ut/python/dataset/test_ngram_op.py

@@ -20,6 +20,24 @@ import mindspore.dataset as ds
 import mindspore.dataset.text as text
 
 
+def test_ngram_callable():
+    """
+    Test ngram op is callable
+    """
+    op = text.Ngram(2, separator="-")
+
+    input1 = " WildRose Country"
+    input1 = np.array(input1.split(" "), dtype='S')
+    expect1 = ['-WildRose', 'WildRose-Country']
+    result1 = op(input1)
+    assert np.array_equal(result1, expect1)
+
+    input2 = ["WildRose Country", "Canada's Ocean Playground", "Land of Living Skies"]
+    expect2 = ["WildRose Country-Canada's Ocean Playground", "Canada's Ocean Playground-Land of Living Skies"]
+    result2 = op(input2)
+    assert np.array_equal(result2, expect2)
+
+
 def test_multiple_ngrams():
     """ test n-gram where n is a list of integers"""
     plates_mottos = ["WildRose Country", "Canada's Ocean Playground", "Land of Living Skies"]
@@ -105,6 +123,7 @@ def test_corner_cases():
 
 
 if __name__ == '__main__':
+    test_ngram_callable()
     test_multiple_ngrams()
     test_simple_ngram()
     test_corner_cases()

tests/ut/python/dataset/test_pair_truncate.py

@@ -30,6 +30,17 @@ def compare(in1, in2, length, out1, out2):
         np.testing.assert_array_equal(out2, d["s2"])
 
 
+def test_callable():
+    op = text.TruncateSequencePair(3)
+    data = [["1", "2", "3"], ["4", "5"]]
+    result_text = op(*data)
+    column1, column2 = op(["1", "2", "3"], ["4", "5"])
+    assert np.array_equal(result_text[0], ['1', '2'])
+    assert np.array_equal(result_text[1], ['4'])
+    assert np.array_equal(column1, ['1', '2'])
+    assert np.array_equal(column2, ['4'])
+
+
 def test_basics():
     compare(in1=[1, 2, 3], in2=[4, 5], length=4, out1=[1, 2], out2=[4, 5])
     compare(in1=[1, 2], in2=[4, 5], length=4, out1=[1, 2], out2=[4, 5])
@@ -59,6 +70,7 @@ def test_exceptions():
 
 
 if __name__ == "__main__":
+    test_callable()
     test_basics()
     test_basics_odd()
     test_basics_str()

tests/ut/python/dataset/test_sliding_window.py

@@ -16,10 +16,34 @@
 Testing SlidingWindow in mindspore.dataset
 """
 import numpy as np
+import pytest
 import mindspore.dataset as ds
 import mindspore.dataset.text as text
 
 
+def test_sliding_window_callable():
+    """
+    Test sliding window op is callable
+    """
+    op = text.SlidingWindow(2, 0)
+
+    input1 = ["大", "家", "早", "上", "好"]
+    expect = np.array([['大', '家'], ['家', '早'], ['早', '上'], ['上', '好']])
+    result = op(input1)
+    assert np.array_equal(result, expect)
+
+    # test 2D input
+    input2 = [["大", "家", "早", "上", "好"]]
+    with pytest.raises(RuntimeError) as info:
+        _ = op(input2)
+    assert "SlidingWindow: SlidingWindow supports 1D input only for now." in str(info.value)
+
+    # test input multiple tensors
+    with pytest.raises(RuntimeError) as info:
+        _ = op(input1, input1)
+    assert "The op is OneToOne, can only accept one tensor as input." in str(info.value)
+
+
 def test_sliding_window_string():
     """ test sliding_window with string type"""
     inputs = [["大", "家", "早", "上", "好"]]
@@ -104,6 +128,7 @@ def test_sliding_window_exception():
 
 
 if __name__ == '__main__':
+    test_sliding_window_callable()
     test_sliding_window_string()
     test_sliding_window_number()
     test_sliding_window_big_width()

tests/ut/python/dataset/test_to_number_op.py

@@ -50,6 +50,12 @@ def test_to_number_eager():
         _ = op(*input_strings)
     assert "The op is OneToOne, can only accept one tensor as input." in str(info.value)
 
+    # test input invalid tensor
+    invalid_input = [["1", "2", "3"], ["4", "5"]]
+    with pytest.raises(RuntimeError) as info:
+        _ = op(invalid_input)
+    assert "Invalid data type." in str(info.value)
+
 
 def test_to_number_typical_case_integral():
     input_strings = [["-121", "14"], ["-2219", "7623"], ["-8162536", "162371864"],