hesham
5 years ago
parent
e2012a1de9
commit
f2462bb00d
@ -0,0 +1,49 @@
|
|||||||
|
/**
|
||||||
|
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#include "dataset/kernels/data/mask_op.h"
|
||||||
|
|
||||||
|
#include "dataset/core/tensor.h"
|
||||||
|
#include "dataset/kernels/tensor_op.h"
|
||||||
|
|
||||||
|
namespace mindspore {
|
||||||
|
namespace dataset {
|
||||||
|
|
||||||
|
Status MaskOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
|
||||||
|
IO_CHECK(input, output);
|
||||||
|
std::shared_ptr<Tensor> temp_output;
|
||||||
|
CHECK_FAIL_RETURN_UNEXPECTED(type_.IsNumeric(), "Cannot generate a string mask. Type should be numeric.");
|
||||||
|
|
||||||
|
RETURN_IF_NOT_OK(Mask(input, &temp_output, value_, op_));
|
||||||
|
|
||||||
|
// cast the output to the the required type. Skip casting if type_ is bool.
|
||||||
|
if (type_ != DataType::DE_BOOL) {
|
||||||
|
RETURN_IF_NOT_OK(cast_->Compute(temp_output, output));
|
||||||
|
} else {
|
||||||
|
*output = temp_output;
|
||||||
|
}
|
||||||
|
|
||||||
|
return Status::OK();
|
||||||
|
}
|
||||||
|
|
||||||
|
Status MaskOp::OutputType(const std::vector<DataType> &inputs, std::vector<DataType> &outputs) {
|
||||||
|
RETURN_IF_NOT_OK(TensorOp::OutputType(inputs, outputs));
|
||||||
|
outputs[0] = type_;
|
||||||
|
return Status::OK();
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace dataset
|
||||||
|
} // namespace mindspore
|
||||||
@ -0,0 +1,54 @@
|
|||||||
|
/**
|
||||||
|
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
#ifndef DATASET_KERNELS_DATA_MASK_OP_H_
|
||||||
|
#define DATASET_KERNELS_DATA_MASK_OP_H_
|
||||||
|
|
||||||
|
#include <algorithm>
|
||||||
|
#include <memory>
|
||||||
|
#include <string>
|
||||||
|
#include <utility>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
#include "dataset/core/tensor.h"
|
||||||
|
#include "dataset/kernels/tensor_op.h"
|
||||||
|
#include "dataset/kernels/data/type_cast_op.h"
|
||||||
|
#include "dataset/kernels/data/data_utils.h"
|
||||||
|
|
||||||
|
namespace mindspore {
|
||||||
|
namespace dataset {
|
||||||
|
|
||||||
|
class MaskOp : public TensorOp {
|
||||||
|
public:
|
||||||
|
MaskOp(RelationalOp op, std::shared_ptr<Tensor> value, DataType type = DataType(DataType::DE_BOOL))
|
||||||
|
: op_(op), value_(std::move(value)), type_(type), cast_(new TypeCastOp(type)) {}
|
||||||
|
|
||||||
|
~MaskOp() override = default;
|
||||||
|
|
||||||
|
void Print(std::ostream &out) const override { out << "MaskOp"; }
|
||||||
|
|
||||||
|
Status Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) override;
|
||||||
|
|
||||||
|
Status OutputType(const std::vector<DataType> &inputs, std::vector<DataType> &outputs) override;
|
||||||
|
|
||||||
|
private:
|
||||||
|
RelationalOp op_;
|
||||||
|
std::shared_ptr<Tensor> value_;
|
||||||
|
DataType type_;
|
||||||
|
std::unique_ptr<TypeCastOp> cast_;
|
||||||
|
};
|
||||||
|
} // namespace dataset
|
||||||
|
} // namespace mindspore
|
||||||
|
#endif // DATASET_KERNELS_DATA_MASK_OP_H_
|
||||||
@ -0,0 +1,63 @@
|
|||||||
|
/**
|
||||||
|
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
#include <memory>
|
||||||
|
#include <string>
|
||||||
|
#include "dataset/core/client.h"
|
||||||
|
#include "common/common.h"
|
||||||
|
#include "gtest/gtest.h"
|
||||||
|
#include "securec.h"
|
||||||
|
#include "dataset/core/tensor.h"
|
||||||
|
#include "dataset/core/cv_tensor.h"
|
||||||
|
#include "dataset/core/data_type.h"
|
||||||
|
#include "dataset/util/de_error.h"
|
||||||
|
#include "dataset/kernels/data/mask_op.h"
|
||||||
|
#include "dataset/kernels/data/data_utils.h"
|
||||||
|
|
||||||
|
using namespace mindspore::dataset;
|
||||||
|
|
||||||
|
namespace py = pybind11;
|
||||||
|
|
||||||
|
class MindDataTestMaskOp : public UT::Common {
|
||||||
|
public:
|
||||||
|
MindDataTestMaskOp() {}
|
||||||
|
|
||||||
|
void SetUp() { GlobalInit(); }
|
||||||
|
};
|
||||||
|
|
||||||
|
TEST_F(MindDataTestMaskOp, Basics) {
|
||||||
|
std::shared_ptr<Tensor> t;
|
||||||
|
Tensor::CreateTensor(&t, std::vector<uint32_t>({1, 2, 3, 4, 5, 6}));
|
||||||
|
std::shared_ptr<Tensor> v;
|
||||||
|
Tensor::CreateTensor(&v, std::vector<uint32_t>({3}), TensorShape::CreateScalar());
|
||||||
|
std::shared_ptr<MaskOp> op = std::make_shared<MaskOp>(RelationalOp::kEqual, v, DataType(DataType::DE_UINT16));
|
||||||
|
std::shared_ptr<Tensor> out;
|
||||||
|
ASSERT_TRUE(op->Compute(t, &out).IsOk());
|
||||||
|
|
||||||
|
op = std::make_shared<MaskOp>(RelationalOp::kNotEqual, v, DataType(DataType::DE_UINT16));
|
||||||
|
ASSERT_TRUE(op->Compute(t, &out).IsOk());
|
||||||
|
|
||||||
|
op = std::make_shared<MaskOp>(RelationalOp::kLessEqual, v, DataType(DataType::DE_UINT16));
|
||||||
|
ASSERT_TRUE(op->Compute(t, &out).IsOk());
|
||||||
|
|
||||||
|
op = std::make_shared<MaskOp>(RelationalOp::kLess, v, DataType(DataType::DE_UINT16));
|
||||||
|
ASSERT_TRUE(op->Compute(t, &out).IsOk());
|
||||||
|
|
||||||
|
op = std::make_shared<MaskOp>(RelationalOp::kGreaterEqual, v, DataType(DataType::DE_UINT16));
|
||||||
|
ASSERT_TRUE(op->Compute(t, &out).IsOk());
|
||||||
|
|
||||||
|
op = std::make_shared<MaskOp>(RelationalOp::kGreater, v, DataType(DataType::DE_UINT16));
|
||||||
|
ASSERT_TRUE(op->Compute(t, &out).IsOk());
|
||||||
|
}
|
||||||
@ -0,0 +1,132 @@
|
|||||||
|
# Copyright 2020 Huawei Technologies Co., Ltd
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
# ==============================================================================
|
||||||
|
"""
|
||||||
|
Testing Mask op in DE
|
||||||
|
"""
|
||||||
|
import numpy as np
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
import mindspore.common.dtype as mstype
|
||||||
|
import mindspore.dataset as ds
|
||||||
|
import mindspore.dataset.transforms.c_transforms as ops
|
||||||
|
|
||||||
|
mstype_to_np_type = {
|
||||||
|
mstype.bool_: np.bool,
|
||||||
|
mstype.int8: np.int8,
|
||||||
|
mstype.uint8: np.uint8,
|
||||||
|
mstype.int16: np.int16,
|
||||||
|
mstype.uint16: np.uint16,
|
||||||
|
mstype.int32: np.int32,
|
||||||
|
mstype.uint32: np.uint32,
|
||||||
|
mstype.int64: np.int64,
|
||||||
|
mstype.uint64: np.uint64,
|
||||||
|
mstype.float16: np.float16,
|
||||||
|
mstype.float32: np.float32,
|
||||||
|
mstype.float64: np.float64,
|
||||||
|
mstype.string: np.str
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def mask_compare(array, op, constant, dtype=mstype.bool_):
|
||||||
|
data = ds.NumpySlicesDataset([array])
|
||||||
|
array = np.array(array)
|
||||||
|
data = data.map(operations=ops.Mask(op, constant, dtype))
|
||||||
|
for d in data:
|
||||||
|
if op == ops.Relational.EQ:
|
||||||
|
array = array == np.array(constant, dtype=array.dtype)
|
||||||
|
elif op == ops.Relational.NE:
|
||||||
|
array = array != np.array(constant, dtype=array.dtype)
|
||||||
|
elif op == ops.Relational.GT:
|
||||||
|
array = array > np.array(constant, dtype=array.dtype)
|
||||||
|
elif op == ops.Relational.GE:
|
||||||
|
array = array >= np.array(constant, dtype=array.dtype)
|
||||||
|
elif op == ops.Relational.LT:
|
||||||
|
array = array < np.array(constant, dtype=array.dtype)
|
||||||
|
elif op == ops.Relational.LE:
|
||||||
|
array = array <= np.array(constant, dtype=array.dtype)
|
||||||
|
|
||||||
|
array = array.astype(dtype=mstype_to_np_type[dtype])
|
||||||
|
|
||||||
|
np.testing.assert_array_equal(array, d[0])
|
||||||
|
|
||||||
|
|
||||||
|
def test_int_comparison():
|
||||||
|
for k in mstype_to_np_type:
|
||||||
|
if k == mstype.string:
|
||||||
|
continue
|
||||||
|
mask_compare([1, 2, 3, 4, 5], ops.Relational.EQ, 3, k)
|
||||||
|
mask_compare([1, 2, 3, 4, 5], ops.Relational.NE, 3, k)
|
||||||
|
mask_compare([1, 2, 3, 4, 5], ops.Relational.LT, 3, k)
|
||||||
|
mask_compare([1, 2, 3, 4, 5], ops.Relational.LE, 3, k)
|
||||||
|
mask_compare([1, 2, 3, 4, 5], ops.Relational.GT, 3, k)
|
||||||
|
mask_compare([1, 2, 3, 4, 5], ops.Relational.GE, 3, k)
|
||||||
|
|
||||||
|
|
||||||
|
def test_float_comparison():
|
||||||
|
for k in mstype_to_np_type:
|
||||||
|
if k == mstype.string:
|
||||||
|
continue
|
||||||
|
mask_compare([1.5, 2.5, 3., 4.5, 5.5], ops.Relational.EQ, 3, k)
|
||||||
|
mask_compare([1.5, 2.5, 3., 4.5, 5.5], ops.Relational.NE, 3, k)
|
||||||
|
mask_compare([1.5, 2.5, 3., 4.5, 5.5], ops.Relational.LT, 3, k)
|
||||||
|
mask_compare([1.5, 2.5, 3., 4.5, 5.5], ops.Relational.LE, 3, k)
|
||||||
|
mask_compare([1.5, 2.5, 3., 4.5, 5.5], ops.Relational.GT, 3, k)
|
||||||
|
mask_compare([1.5, 2.5, 3., 4.5, 5.5], ops.Relational.GE, 3, k)
|
||||||
|
|
||||||
|
|
||||||
|
def test_float_comparison2():
|
||||||
|
for k in mstype_to_np_type:
|
||||||
|
if k == mstype.string:
|
||||||
|
continue
|
||||||
|
mask_compare([1, 2, 3, 4, 5], ops.Relational.EQ, 3.5, k)
|
||||||
|
mask_compare([1, 2, 3, 4, 5], ops.Relational.NE, 3.5, k)
|
||||||
|
mask_compare([1, 2, 3, 4, 5], ops.Relational.LT, 3.5, k)
|
||||||
|
mask_compare([1, 2, 3, 4, 5], ops.Relational.LE, 3.5, k)
|
||||||
|
mask_compare([1, 2, 3, 4, 5], ops.Relational.GT, 3.5, k)
|
||||||
|
mask_compare([1, 2, 3, 4, 5], ops.Relational.GE, 3.5, k)
|
||||||
|
|
||||||
|
|
||||||
|
def test_string_comparison():
|
||||||
|
for k in mstype_to_np_type:
|
||||||
|
if k == mstype.string:
|
||||||
|
continue
|
||||||
|
mask_compare(["1.5", "2.5", "3.", "4.5", "5.5"], ops.Relational.EQ, "3.", k)
|
||||||
|
mask_compare(["1.5", "2.5", "3.", "4.5", "5.5"], ops.Relational.NE, "3.", k)
|
||||||
|
mask_compare(["1.5", "2.5", "3.", "4.5", "5.5"], ops.Relational.LT, "3.", k)
|
||||||
|
mask_compare(["1.5", "2.5", "3.", "4.5", "5.5"], ops.Relational.LE, "3.", k)
|
||||||
|
mask_compare(["1.5", "2.5", "3.", "4.5", "5.5"], ops.Relational.GT, "3.", k)
|
||||||
|
mask_compare(["1.5", "2.5", "3.", "4.5", "5.5"], ops.Relational.GE, "3.", k)
|
||||||
|
|
||||||
|
|
||||||
|
def test_mask_exceptions_str():
|
||||||
|
with pytest.raises(RuntimeError) as info:
|
||||||
|
mask_compare([1, 2, 3, 4, 5], ops.Relational.EQ, "3.5")
|
||||||
|
assert "Cannot convert constant value to the type of the input tensor." in str(info.value)
|
||||||
|
|
||||||
|
with pytest.raises(RuntimeError) as info:
|
||||||
|
mask_compare(["1", "2", "3", "4", "5"], ops.Relational.EQ, 3.5)
|
||||||
|
assert "Cannot convert constant value to the type of the input tensor." in str(info.value)
|
||||||
|
|
||||||
|
with pytest.raises(RuntimeError) as info:
|
||||||
|
mask_compare(["1", "2", "3", "4", "5"], ops.Relational.EQ, "3.5", mstype.string)
|
||||||
|
assert "Cannot generate a string mask. Type should be numeric." in str(info.value)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
test_int_comparison()
|
||||||
|
test_float_comparison()
|
||||||
|
test_float_comparison2()
|
||||||
|
test_string_comparison()
|
||||||
|
test_mask_exceptions_str()
|
||||||
Loading…
Reference in new issue