remove graphengine changes concat op Truncate Pair concat_op remove graph engine changespull/2253/head
nhussain
5 years ago
parent
ffc8a3c362
commit
5515016dba
@ -1,12 +1,13 @@
|
||||
file(GLOB_RECURSE _CURRENT_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
|
||||
set_property(SOURCE ${_CURRENT_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_MD)
|
||||
add_library(kernels-data OBJECT
|
||||
data_utils.cc
|
||||
one_hot_op.cc
|
||||
pad_end_op.cc
|
||||
type_cast_op.cc
|
||||
to_float16_op.cc
|
||||
fill_op.cc
|
||||
slice_op.cc
|
||||
mask_op.cc
|
||||
)
|
||||
data_utils.cc
|
||||
one_hot_op.cc
|
||||
pad_end_op.cc
|
||||
type_cast_op.cc
|
||||
to_float16_op.cc
|
||||
fill_op.cc
|
||||
slice_op.cc
|
||||
mask_op.cc
|
||||
concatenate_op.cc
|
||||
)
|
||||
|
||||
@ -0,0 +1,55 @@
|
||||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
#include "dataset/kernels/data/concatenate_op.h"
|
||||
|
||||
#include "dataset/core/tensor.h"
|
||||
#include "dataset/kernels/data/data_utils.h"
|
||||
#include "dataset/kernels/tensor_op.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace dataset {
|
||||
|
||||
Status ConcatenateOp::Compute(const TensorRow &input, TensorRow *output) {
|
||||
IO_CHECK_VECTOR(input, output);
|
||||
RETURN_IF_NOT_OK(Concatenate(input, output, axis_, prepend_, append_));
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status ConcatenateOp::OutputShape(const std::vector<TensorShape> &inputs, std::vector<TensorShape> &outputs) {
|
||||
RETURN_IF_NOT_OK(TensorOp::OutputShape(inputs, outputs));
|
||||
|
||||
std::vector<TensorShape> inputs_copy;
|
||||
inputs_copy.push_back(inputs[0].Squeeze());
|
||||
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(inputs.at(0).Rank() == 1, "Only 1D input tensors supported");
|
||||
|
||||
outputs.clear();
|
||||
dsize_t output_shape = 0;
|
||||
output_shape = output_shape + inputs.at(0).NumOfElements();
|
||||
if (prepend_ != nullptr) {
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(prepend_->shape().Rank() == 1, "Only 1D prepend tensors supported");
|
||||
output_shape = output_shape + prepend_->shape().NumOfElements();
|
||||
}
|
||||
if (append_ != nullptr) {
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(append_->shape().Rank() == 1, "Only 1D append tensors supported");
|
||||
output_shape = output_shape + append_->shape().NumOfElements();
|
||||
}
|
||||
|
||||
outputs.emplace_back(std::vector<dsize_t>{output_shape});
|
||||
return Status::OK();
|
||||
}
|
||||
} // namespace dataset
|
||||
} // namespace mindspore
|
||||
@ -0,0 +1,66 @@
|
||||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef DATASET_KERNELS_DATA_CONCATENATE_OP_H_
|
||||
#define DATASET_KERNELS_DATA_CONCATENATE_OP_H_
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <memory>
|
||||
|
||||
#include "dataset/core/tensor.h"
|
||||
#include "dataset/kernels/tensor_op.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace dataset {
|
||||
|
||||
class ConcatenateOp : public TensorOp {
|
||||
public:
|
||||
/// Constructor to ConcatenateOp.
|
||||
/// @param int8_t axis - axis to concatenate tensors along.
|
||||
/// @param std::shared_ptr<Tensor> prepend - prepend tensor.
|
||||
/// @param std::shared_ptr<Tensor> append -append tensor.
|
||||
explicit ConcatenateOp(int8_t axis, std::shared_ptr<Tensor> prepend, std::shared_ptr<Tensor> append)
|
||||
: axis_(axis), prepend_(prepend), append_(append) {}
|
||||
|
||||
~ConcatenateOp() override = default;
|
||||
|
||||
/// Print method to see which tensor Op this is.
|
||||
/// @param std::ostream &out - output stream object.
|
||||
void Print(std::ostream &out) const override { out << "ConcatenateOp"; }
|
||||
|
||||
/// Compute method allowing multiple tensors as inputs
|
||||
/// @param TensorRow &input - input tensor rows
|
||||
/// @param TensorRow *output - output tensor rows
|
||||
Status Compute(const TensorRow &input, TensorRow *output) override;
|
||||
|
||||
/// Compute tensor output shape
|
||||
/// @param std::vector<TensorShape> &inputs - vector of input tensor shapes
|
||||
/// @param std::vector<TensorShape< &outputs - vector of output tensor shapes
|
||||
Status OutputShape(const std::vector<TensorShape> &inputs, std::vector<TensorShape> &outputs) override;
|
||||
|
||||
/// Number of inputs the tensor operation accepts
|
||||
uint32_t NumInput() override { return 0; }
|
||||
|
||||
private:
|
||||
int8_t axis_;
|
||||
std::shared_ptr<Tensor> prepend_;
|
||||
std::shared_ptr<Tensor> append_;
|
||||
};
|
||||
} // namespace dataset
|
||||
} // namespace mindspore
|
||||
|
||||
#endif // MINDSPORE_CONCATENATE_OP_H
|
||||
@ -0,0 +1,66 @@
|
||||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
#include "common/common.h"
|
||||
#include "dataset/kernels/data/concatenate_op.h"
|
||||
#include "utils/log_adapter.h"
|
||||
|
||||
using namespace mindspore::dataset;
|
||||
using mindspore::LogStream;
|
||||
using mindspore::ExceptionType::NoExceptionType;
|
||||
using mindspore::MsLogLevel::INFO;
|
||||
|
||||
class MindDataTestConcatenateOp : public UT::Common {
|
||||
protected:
|
||||
MindDataTestConcatenateOp() {}
|
||||
};
|
||||
|
||||
TEST_F(MindDataTestConcatenateOp, TestOp) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestConcatenate-TestOp.";
|
||||
uint64_t labels[3] = {1, 1, 2};
|
||||
TensorShape shape({3});
|
||||
std::shared_ptr<Tensor> input =
|
||||
std::make_shared<Tensor>(shape, DataType(DataType::DE_UINT64), reinterpret_cast<unsigned char *>(labels));
|
||||
|
||||
uint64_t append_labels[3] = {4, 4, 4};
|
||||
std::shared_ptr<Tensor> append =
|
||||
std::make_shared<Tensor>(shape, DataType(DataType::DE_UINT64), reinterpret_cast<unsigned char *>(append_labels));
|
||||
|
||||
std::shared_ptr<Tensor> output;
|
||||
std::unique_ptr<ConcatenateOp> op(new ConcatenateOp(0, nullptr, append));
|
||||
TensorRow in;
|
||||
in.push_back(input);
|
||||
TensorRow out_row;
|
||||
Status s = op->Compute(in, &out_row);
|
||||
uint64_t out[6] = {1, 1, 2, 4, 4, 4};
|
||||
|
||||
std::shared_ptr<Tensor> expected =
|
||||
std::make_shared<Tensor>(TensorShape{6}, DataType(DataType::DE_UINT64), reinterpret_cast<unsigned char *>(out));
|
||||
output = out_row[0];
|
||||
EXPECT_TRUE(s.IsOk());
|
||||
ASSERT_TRUE(output->shape() == expected->shape());
|
||||
ASSERT_TRUE(output->type() == expected->type());
|
||||
MS_LOG(DEBUG) << *output << std::endl;
|
||||
MS_LOG(DEBUG) << *expected << std::endl;
|
||||
|
||||
ASSERT_TRUE(*output == *expected);
|
||||
|
||||
// std::vector<TensorShape> inputs = {TensorShape({3})};
|
||||
// std::vector<TensorShape> outputs = {};
|
||||
// s = op->OutputShape(inputs, outputs);
|
||||
// EXPECT_TRUE(s.IsOk());
|
||||
// ASSERT_TRUE(outputs[0] == TensorShape{6});
|
||||
// MS_LOG(INFO) << "MindDataTestConcatenateOp-TestOp end.";
|
||||
}
|
||||
@ -0,0 +1,175 @@
|
||||
# Copyright 2020 Huawei Technologies Co., Ltd
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""
|
||||
Testing concatenate op
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
import mindspore.dataset as ds
|
||||
import mindspore.dataset.transforms.c_transforms as data_trans
|
||||
|
||||
|
||||
def test_concatenate_op_all():
|
||||
def gen():
|
||||
yield (np.array([5., 6., 7., 8.], dtype=np.float),)
|
||||
|
||||
prepend_tensor = np.array([1.4, 2., 3., 4., 4.5], dtype=np.float)
|
||||
append_tensor = np.array([9., 10.3, 11., 12.], dtype=np.float)
|
||||
data = ds.GeneratorDataset(gen, column_names=["col"])
|
||||
concatenate_op = data_trans.Concatenate(0, prepend_tensor, append_tensor)
|
||||
data = data.map(input_columns=["col"], operations=concatenate_op)
|
||||
expected = np.array([1.4, 2., 3., 4., 4.5, 5., 6., 7., 8., 9., 10.3,
|
||||
11., 12.])
|
||||
for data_row in data:
|
||||
np.testing.assert_array_equal(data_row[0], expected)
|
||||
|
||||
|
||||
def test_concatenate_op_none():
|
||||
def gen():
|
||||
yield (np.array([5., 6., 7., 8.], dtype=np.float),)
|
||||
|
||||
data = ds.GeneratorDataset(gen, column_names=["col"])
|
||||
concatenate_op = data_trans.Concatenate()
|
||||
|
||||
data = data.map(input_columns=["col"], operations=concatenate_op)
|
||||
for data_row in data:
|
||||
np.testing.assert_array_equal(data_row[0], np.array([5., 6., 7., 8.], dtype=np.float))
|
||||
|
||||
|
||||
def test_concatenate_op_string():
|
||||
def gen():
|
||||
yield (np.array(["ss", "ad"], dtype='S'),)
|
||||
|
||||
prepend_tensor = np.array(["dw", "df"], dtype='S')
|
||||
append_tensor = np.array(["dwsdf", "df"], dtype='S')
|
||||
data = ds.GeneratorDataset(gen, column_names=["col"])
|
||||
concatenate_op = data_trans.Concatenate(0, prepend_tensor, append_tensor)
|
||||
|
||||
data = data.map(input_columns=["col"], operations=concatenate_op)
|
||||
expected = np.array(["dw", "df", "ss", "ad", "dwsdf", "df"], dtype='S')
|
||||
for data_row in data:
|
||||
np.testing.assert_array_equal(data_row[0], expected)
|
||||
|
||||
|
||||
def test_concatenate_op_multi_input_string():
|
||||
prepend_tensor = np.array(["dw", "df"], dtype='S')
|
||||
append_tensor = np.array(["dwsdf", "df"], dtype='S')
|
||||
|
||||
data = ([["1", "2", "d"]], [["3", "4", "e"]])
|
||||
data = ds.NumpySlicesDataset(data, column_names=["col1", "col2"])
|
||||
|
||||
concatenate_op = data_trans.Concatenate(0, prepend=prepend_tensor, append=append_tensor)
|
||||
|
||||
data = data.map(input_columns=["col1", "col2"], columns_order=["out1"], output_columns=["out1"],
|
||||
operations=concatenate_op)
|
||||
expected = np.array(["dw", "df", "1", "2", "d", "3", "4", "e", "dwsdf", "df"], dtype='S')
|
||||
for data_row in data:
|
||||
np.testing.assert_array_equal(data_row[0], expected)
|
||||
|
||||
|
||||
def test_concatenate_op_multi_input_numeric():
|
||||
prepend_tensor = np.array([3, 5])
|
||||
|
||||
data = ([[1, 2]], [[3, 4]])
|
||||
data = ds.NumpySlicesDataset(data, column_names=["col1", "col2"])
|
||||
|
||||
concatenate_op = data_trans.Concatenate(0, prepend=prepend_tensor)
|
||||
|
||||
data = data.map(input_columns=["col1", "col2"], columns_order=["out1"], output_columns=["out1"],
|
||||
operations=concatenate_op)
|
||||
expected = np.array([3, 5, 1, 2, 3, 4])
|
||||
for data_row in data:
|
||||
np.testing.assert_array_equal(data_row[0], expected)
|
||||
|
||||
|
||||
def test_concatenate_op_type_mismatch():
|
||||
def gen():
|
||||
yield (np.array([3, 4], dtype=np.float),)
|
||||
|
||||
prepend_tensor = np.array(["ss", "ad"], dtype='S')
|
||||
data = ds.GeneratorDataset(gen, column_names=["col"])
|
||||
concatenate_op = data_trans.Concatenate(0, prepend_tensor)
|
||||
|
||||
data = data.map(input_columns=["col"], operations=concatenate_op)
|
||||
with pytest.raises(RuntimeError) as error_info:
|
||||
for _ in data:
|
||||
pass
|
||||
assert "Tensor types do not match" in repr(error_info.value)
|
||||
|
||||
|
||||
def test_concatenate_op_type_mismatch2():
|
||||
def gen():
|
||||
yield (np.array(["ss", "ad"], dtype='S'),)
|
||||
|
||||
prepend_tensor = np.array([3, 5], dtype=np.float)
|
||||
data = ds.GeneratorDataset(gen, column_names=["col"])
|
||||
concatenate_op = data_trans.Concatenate(0, prepend_tensor)
|
||||
|
||||
data = data.map(input_columns=["col"], operations=concatenate_op)
|
||||
with pytest.raises(RuntimeError) as error_info:
|
||||
for _ in data:
|
||||
pass
|
||||
assert "Tensor types do not match" in repr(error_info.value)
|
||||
|
||||
|
||||
def test_concatenate_op_incorrect_dim():
|
||||
def gen():
|
||||
yield (np.array([["ss", "ad"], ["ss", "ad"]], dtype='S'),)
|
||||
|
||||
prepend_tensor = np.array([3, 5], dtype=np.float)
|
||||
concatenate_op = data_trans.Concatenate(0, prepend_tensor)
|
||||
data = ds.GeneratorDataset(gen, column_names=["col"])
|
||||
|
||||
data = data.map(input_columns=["col"], operations=concatenate_op)
|
||||
with pytest.raises(RuntimeError) as error_info:
|
||||
for _ in data:
|
||||
pass
|
||||
assert "Only 1D tensors supported" in repr(error_info.value)
|
||||
|
||||
|
||||
def test_concatenate_op_wrong_axis():
|
||||
with pytest.raises(ValueError) as error_info:
|
||||
data_trans.Concatenate(2)
|
||||
assert "only 1D concatenation supported." in repr(error_info.value)
|
||||
|
||||
|
||||
def test_concatenate_op_incorrect_input_dim():
|
||||
def gen():
|
||||
yield (np.array(["ss", "ad"], dtype='S'),)
|
||||
|
||||
prepend_tensor = np.array([["ss", "ad"], ["ss", "ad"]], dtype='S')
|
||||
data = ds.GeneratorDataset(gen, column_names=["col"])
|
||||
concatenate_op = data_trans.Concatenate(0, prepend_tensor)
|
||||
|
||||
data = data.map(input_columns=["col"], operations=concatenate_op)
|
||||
with pytest.raises(RuntimeError) as error_info:
|
||||
for _ in data:
|
||||
pass
|
||||
assert "Only 1D tensors supported" in repr(error_info.value)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_concatenate_op_all()
|
||||
test_concatenate_op_none()
|
||||
test_concatenate_op_string()
|
||||
test_concatenate_op_type_mismatch()
|
||||
test_concatenate_op_type_mismatch2()
|
||||
test_concatenate_op_incorrect_dim()
|
||||
test_concatenate_op_incorrect_input_dim()
|
||||
test_concatenate_op_multi_input_numeric()
|
||||
test_concatenate_op_multi_input_string()
|
||||
test_concatenate_op_wrong_axis()
|
||||
Loading…
Reference in new issue