hesham
5 years ago
parent
e2012a1de9
commit
f2462bb00d
@ -0,0 +1,49 @@
|
||||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "dataset/kernels/data/mask_op.h"
|
||||
|
||||
#include "dataset/core/tensor.h"
|
||||
#include "dataset/kernels/tensor_op.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace dataset {
|
||||
|
||||
Status MaskOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
|
||||
IO_CHECK(input, output);
|
||||
std::shared_ptr<Tensor> temp_output;
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(type_.IsNumeric(), "Cannot generate a string mask. Type should be numeric.");
|
||||
|
||||
RETURN_IF_NOT_OK(Mask(input, &temp_output, value_, op_));
|
||||
|
||||
// cast the output to the the required type. Skip casting if type_ is bool.
|
||||
if (type_ != DataType::DE_BOOL) {
|
||||
RETURN_IF_NOT_OK(cast_->Compute(temp_output, output));
|
||||
} else {
|
||||
*output = temp_output;
|
||||
}
|
||||
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status MaskOp::OutputType(const std::vector<DataType> &inputs, std::vector<DataType> &outputs) {
|
||||
RETURN_IF_NOT_OK(TensorOp::OutputType(inputs, outputs));
|
||||
outputs[0] = type_;
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
} // namespace dataset
|
||||
} // namespace mindspore
|
||||
@ -0,0 +1,54 @@
|
||||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
#ifndef DATASET_KERNELS_DATA_MASK_OP_H_
|
||||
#define DATASET_KERNELS_DATA_MASK_OP_H_
|
||||
|
||||
#include <algorithm>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "dataset/core/tensor.h"
|
||||
#include "dataset/kernels/tensor_op.h"
|
||||
#include "dataset/kernels/data/type_cast_op.h"
|
||||
#include "dataset/kernels/data/data_utils.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace dataset {
|
||||
|
||||
class MaskOp : public TensorOp {
|
||||
public:
|
||||
MaskOp(RelationalOp op, std::shared_ptr<Tensor> value, DataType type = DataType(DataType::DE_BOOL))
|
||||
: op_(op), value_(std::move(value)), type_(type), cast_(new TypeCastOp(type)) {}
|
||||
|
||||
~MaskOp() override = default;
|
||||
|
||||
void Print(std::ostream &out) const override { out << "MaskOp"; }
|
||||
|
||||
Status Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) override;
|
||||
|
||||
Status OutputType(const std::vector<DataType> &inputs, std::vector<DataType> &outputs) override;
|
||||
|
||||
private:
|
||||
RelationalOp op_;
|
||||
std::shared_ptr<Tensor> value_;
|
||||
DataType type_;
|
||||
std::unique_ptr<TypeCastOp> cast_;
|
||||
};
|
||||
} // namespace dataset
|
||||
} // namespace mindspore
|
||||
#endif // DATASET_KERNELS_DATA_MASK_OP_H_
|
||||
@ -0,0 +1,63 @@
|
||||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include "dataset/core/client.h"
|
||||
#include "common/common.h"
|
||||
#include "gtest/gtest.h"
|
||||
#include "securec.h"
|
||||
#include "dataset/core/tensor.h"
|
||||
#include "dataset/core/cv_tensor.h"
|
||||
#include "dataset/core/data_type.h"
|
||||
#include "dataset/util/de_error.h"
|
||||
#include "dataset/kernels/data/mask_op.h"
|
||||
#include "dataset/kernels/data/data_utils.h"
|
||||
|
||||
using namespace mindspore::dataset;
|
||||
|
||||
namespace py = pybind11;
|
||||
|
||||
class MindDataTestMaskOp : public UT::Common {
|
||||
public:
|
||||
MindDataTestMaskOp() {}
|
||||
|
||||
void SetUp() { GlobalInit(); }
|
||||
};
|
||||
|
||||
TEST_F(MindDataTestMaskOp, Basics) {
|
||||
std::shared_ptr<Tensor> t;
|
||||
Tensor::CreateTensor(&t, std::vector<uint32_t>({1, 2, 3, 4, 5, 6}));
|
||||
std::shared_ptr<Tensor> v;
|
||||
Tensor::CreateTensor(&v, std::vector<uint32_t>({3}), TensorShape::CreateScalar());
|
||||
std::shared_ptr<MaskOp> op = std::make_shared<MaskOp>(RelationalOp::kEqual, v, DataType(DataType::DE_UINT16));
|
||||
std::shared_ptr<Tensor> out;
|
||||
ASSERT_TRUE(op->Compute(t, &out).IsOk());
|
||||
|
||||
op = std::make_shared<MaskOp>(RelationalOp::kNotEqual, v, DataType(DataType::DE_UINT16));
|
||||
ASSERT_TRUE(op->Compute(t, &out).IsOk());
|
||||
|
||||
op = std::make_shared<MaskOp>(RelationalOp::kLessEqual, v, DataType(DataType::DE_UINT16));
|
||||
ASSERT_TRUE(op->Compute(t, &out).IsOk());
|
||||
|
||||
op = std::make_shared<MaskOp>(RelationalOp::kLess, v, DataType(DataType::DE_UINT16));
|
||||
ASSERT_TRUE(op->Compute(t, &out).IsOk());
|
||||
|
||||
op = std::make_shared<MaskOp>(RelationalOp::kGreaterEqual, v, DataType(DataType::DE_UINT16));
|
||||
ASSERT_TRUE(op->Compute(t, &out).IsOk());
|
||||
|
||||
op = std::make_shared<MaskOp>(RelationalOp::kGreater, v, DataType(DataType::DE_UINT16));
|
||||
ASSERT_TRUE(op->Compute(t, &out).IsOk());
|
||||
}
|
||||
@ -0,0 +1,132 @@
|
||||
# Copyright 2020 Huawei Technologies Co., Ltd
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""
|
||||
Testing Mask op in DE
|
||||
"""
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
import mindspore.common.dtype as mstype
|
||||
import mindspore.dataset as ds
|
||||
import mindspore.dataset.transforms.c_transforms as ops
|
||||
|
||||
mstype_to_np_type = {
|
||||
mstype.bool_: np.bool,
|
||||
mstype.int8: np.int8,
|
||||
mstype.uint8: np.uint8,
|
||||
mstype.int16: np.int16,
|
||||
mstype.uint16: np.uint16,
|
||||
mstype.int32: np.int32,
|
||||
mstype.uint32: np.uint32,
|
||||
mstype.int64: np.int64,
|
||||
mstype.uint64: np.uint64,
|
||||
mstype.float16: np.float16,
|
||||
mstype.float32: np.float32,
|
||||
mstype.float64: np.float64,
|
||||
mstype.string: np.str
|
||||
}
|
||||
|
||||
|
||||
def mask_compare(array, op, constant, dtype=mstype.bool_):
|
||||
data = ds.NumpySlicesDataset([array])
|
||||
array = np.array(array)
|
||||
data = data.map(operations=ops.Mask(op, constant, dtype))
|
||||
for d in data:
|
||||
if op == ops.Relational.EQ:
|
||||
array = array == np.array(constant, dtype=array.dtype)
|
||||
elif op == ops.Relational.NE:
|
||||
array = array != np.array(constant, dtype=array.dtype)
|
||||
elif op == ops.Relational.GT:
|
||||
array = array > np.array(constant, dtype=array.dtype)
|
||||
elif op == ops.Relational.GE:
|
||||
array = array >= np.array(constant, dtype=array.dtype)
|
||||
elif op == ops.Relational.LT:
|
||||
array = array < np.array(constant, dtype=array.dtype)
|
||||
elif op == ops.Relational.LE:
|
||||
array = array <= np.array(constant, dtype=array.dtype)
|
||||
|
||||
array = array.astype(dtype=mstype_to_np_type[dtype])
|
||||
|
||||
np.testing.assert_array_equal(array, d[0])
|
||||
|
||||
|
||||
def test_int_comparison():
|
||||
for k in mstype_to_np_type:
|
||||
if k == mstype.string:
|
||||
continue
|
||||
mask_compare([1, 2, 3, 4, 5], ops.Relational.EQ, 3, k)
|
||||
mask_compare([1, 2, 3, 4, 5], ops.Relational.NE, 3, k)
|
||||
mask_compare([1, 2, 3, 4, 5], ops.Relational.LT, 3, k)
|
||||
mask_compare([1, 2, 3, 4, 5], ops.Relational.LE, 3, k)
|
||||
mask_compare([1, 2, 3, 4, 5], ops.Relational.GT, 3, k)
|
||||
mask_compare([1, 2, 3, 4, 5], ops.Relational.GE, 3, k)
|
||||
|
||||
|
||||
def test_float_comparison():
|
||||
for k in mstype_to_np_type:
|
||||
if k == mstype.string:
|
||||
continue
|
||||
mask_compare([1.5, 2.5, 3., 4.5, 5.5], ops.Relational.EQ, 3, k)
|
||||
mask_compare([1.5, 2.5, 3., 4.5, 5.5], ops.Relational.NE, 3, k)
|
||||
mask_compare([1.5, 2.5, 3., 4.5, 5.5], ops.Relational.LT, 3, k)
|
||||
mask_compare([1.5, 2.5, 3., 4.5, 5.5], ops.Relational.LE, 3, k)
|
||||
mask_compare([1.5, 2.5, 3., 4.5, 5.5], ops.Relational.GT, 3, k)
|
||||
mask_compare([1.5, 2.5, 3., 4.5, 5.5], ops.Relational.GE, 3, k)
|
||||
|
||||
|
||||
def test_float_comparison2():
|
||||
for k in mstype_to_np_type:
|
||||
if k == mstype.string:
|
||||
continue
|
||||
mask_compare([1, 2, 3, 4, 5], ops.Relational.EQ, 3.5, k)
|
||||
mask_compare([1, 2, 3, 4, 5], ops.Relational.NE, 3.5, k)
|
||||
mask_compare([1, 2, 3, 4, 5], ops.Relational.LT, 3.5, k)
|
||||
mask_compare([1, 2, 3, 4, 5], ops.Relational.LE, 3.5, k)
|
||||
mask_compare([1, 2, 3, 4, 5], ops.Relational.GT, 3.5, k)
|
||||
mask_compare([1, 2, 3, 4, 5], ops.Relational.GE, 3.5, k)
|
||||
|
||||
|
||||
def test_string_comparison():
|
||||
for k in mstype_to_np_type:
|
||||
if k == mstype.string:
|
||||
continue
|
||||
mask_compare(["1.5", "2.5", "3.", "4.5", "5.5"], ops.Relational.EQ, "3.", k)
|
||||
mask_compare(["1.5", "2.5", "3.", "4.5", "5.5"], ops.Relational.NE, "3.", k)
|
||||
mask_compare(["1.5", "2.5", "3.", "4.5", "5.5"], ops.Relational.LT, "3.", k)
|
||||
mask_compare(["1.5", "2.5", "3.", "4.5", "5.5"], ops.Relational.LE, "3.", k)
|
||||
mask_compare(["1.5", "2.5", "3.", "4.5", "5.5"], ops.Relational.GT, "3.", k)
|
||||
mask_compare(["1.5", "2.5", "3.", "4.5", "5.5"], ops.Relational.GE, "3.", k)
|
||||
|
||||
|
||||
def test_mask_exceptions_str():
|
||||
with pytest.raises(RuntimeError) as info:
|
||||
mask_compare([1, 2, 3, 4, 5], ops.Relational.EQ, "3.5")
|
||||
assert "Cannot convert constant value to the type of the input tensor." in str(info.value)
|
||||
|
||||
with pytest.raises(RuntimeError) as info:
|
||||
mask_compare(["1", "2", "3", "4", "5"], ops.Relational.EQ, 3.5)
|
||||
assert "Cannot convert constant value to the type of the input tensor." in str(info.value)
|
||||
|
||||
with pytest.raises(RuntimeError) as info:
|
||||
mask_compare(["1", "2", "3", "4", "5"], ops.Relational.EQ, "3.5", mstype.string)
|
||||
assert "Cannot generate a string mask. Type should be numeric." in str(info.value)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_int_comparison()
|
||||
test_float_comparison()
|
||||
test_float_comparison2()
|
||||
test_string_comparison()
|
||||
test_mask_exceptions_str()
|
||||
Loading…
Reference in new issue