remove graphengine changes concat op Truncate Pair concat_op remove graph engine changespull/2253/head
nhussain
5 years ago
parent
ffc8a3c362
commit
5515016dba
@ -1,12 +1,13 @@
|
|||||||
file(GLOB_RECURSE _CURRENT_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
|
file(GLOB_RECURSE _CURRENT_SRC_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc")
|
||||||
set_property(SOURCE ${_CURRENT_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_MD)
|
set_property(SOURCE ${_CURRENT_SRC_FILES} PROPERTY COMPILE_DEFINITIONS SUBMODULE_ID=mindspore::SubModuleId::SM_MD)
|
||||||
add_library(kernels-data OBJECT
|
add_library(kernels-data OBJECT
|
||||||
data_utils.cc
|
data_utils.cc
|
||||||
one_hot_op.cc
|
one_hot_op.cc
|
||||||
pad_end_op.cc
|
pad_end_op.cc
|
||||||
type_cast_op.cc
|
type_cast_op.cc
|
||||||
to_float16_op.cc
|
to_float16_op.cc
|
||||||
fill_op.cc
|
fill_op.cc
|
||||||
slice_op.cc
|
slice_op.cc
|
||||||
mask_op.cc
|
mask_op.cc
|
||||||
)
|
concatenate_op.cc
|
||||||
|
)
|
||||||
|
|||||||
@ -0,0 +1,55 @@
|
|||||||
|
/**
|
||||||
|
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
#include "dataset/kernels/data/concatenate_op.h"
|
||||||
|
|
||||||
|
#include "dataset/core/tensor.h"
|
||||||
|
#include "dataset/kernels/data/data_utils.h"
|
||||||
|
#include "dataset/kernels/tensor_op.h"
|
||||||
|
|
||||||
|
namespace mindspore {
|
||||||
|
namespace dataset {
|
||||||
|
|
||||||
|
Status ConcatenateOp::Compute(const TensorRow &input, TensorRow *output) {
|
||||||
|
IO_CHECK_VECTOR(input, output);
|
||||||
|
RETURN_IF_NOT_OK(Concatenate(input, output, axis_, prepend_, append_));
|
||||||
|
return Status::OK();
|
||||||
|
}
|
||||||
|
|
||||||
|
Status ConcatenateOp::OutputShape(const std::vector<TensorShape> &inputs, std::vector<TensorShape> &outputs) {
|
||||||
|
RETURN_IF_NOT_OK(TensorOp::OutputShape(inputs, outputs));
|
||||||
|
|
||||||
|
std::vector<TensorShape> inputs_copy;
|
||||||
|
inputs_copy.push_back(inputs[0].Squeeze());
|
||||||
|
|
||||||
|
CHECK_FAIL_RETURN_UNEXPECTED(inputs.at(0).Rank() == 1, "Only 1D input tensors supported");
|
||||||
|
|
||||||
|
outputs.clear();
|
||||||
|
dsize_t output_shape = 0;
|
||||||
|
output_shape = output_shape + inputs.at(0).NumOfElements();
|
||||||
|
if (prepend_ != nullptr) {
|
||||||
|
CHECK_FAIL_RETURN_UNEXPECTED(prepend_->shape().Rank() == 1, "Only 1D prepend tensors supported");
|
||||||
|
output_shape = output_shape + prepend_->shape().NumOfElements();
|
||||||
|
}
|
||||||
|
if (append_ != nullptr) {
|
||||||
|
CHECK_FAIL_RETURN_UNEXPECTED(append_->shape().Rank() == 1, "Only 1D append tensors supported");
|
||||||
|
output_shape = output_shape + append_->shape().NumOfElements();
|
||||||
|
}
|
||||||
|
|
||||||
|
outputs.emplace_back(std::vector<dsize_t>{output_shape});
|
||||||
|
return Status::OK();
|
||||||
|
}
|
||||||
|
} // namespace dataset
|
||||||
|
} // namespace mindspore
|
||||||
@ -0,0 +1,66 @@
|
|||||||
|
/**
|
||||||
|
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#ifndef DATASET_KERNELS_DATA_CONCATENATE_OP_H_
|
||||||
|
#define DATASET_KERNELS_DATA_CONCATENATE_OP_H_
|
||||||
|
|
||||||
|
#include <string>
|
||||||
|
#include <vector>
|
||||||
|
#include <memory>
|
||||||
|
|
||||||
|
#include "dataset/core/tensor.h"
|
||||||
|
#include "dataset/kernels/tensor_op.h"
|
||||||
|
|
||||||
|
namespace mindspore {
|
||||||
|
namespace dataset {
|
||||||
|
|
||||||
|
class ConcatenateOp : public TensorOp {
|
||||||
|
public:
|
||||||
|
/// Constructor to ConcatenateOp.
|
||||||
|
/// @param int8_t axis - axis to concatenate tensors along.
|
||||||
|
/// @param std::shared_ptr<Tensor> prepend - prepend tensor.
|
||||||
|
/// @param std::shared_ptr<Tensor> append -append tensor.
|
||||||
|
explicit ConcatenateOp(int8_t axis, std::shared_ptr<Tensor> prepend, std::shared_ptr<Tensor> append)
|
||||||
|
: axis_(axis), prepend_(prepend), append_(append) {}
|
||||||
|
|
||||||
|
~ConcatenateOp() override = default;
|
||||||
|
|
||||||
|
/// Print method to see which tensor Op this is.
|
||||||
|
/// @param std::ostream &out - output stream object.
|
||||||
|
void Print(std::ostream &out) const override { out << "ConcatenateOp"; }
|
||||||
|
|
||||||
|
/// Compute method allowing multiple tensors as inputs
|
||||||
|
/// @param TensorRow &input - input tensor rows
|
||||||
|
/// @param TensorRow *output - output tensor rows
|
||||||
|
Status Compute(const TensorRow &input, TensorRow *output) override;
|
||||||
|
|
||||||
|
/// Compute tensor output shape
|
||||||
|
/// @param std::vector<TensorShape> &inputs - vector of input tensor shapes
|
||||||
|
/// @param std::vector<TensorShape< &outputs - vector of output tensor shapes
|
||||||
|
Status OutputShape(const std::vector<TensorShape> &inputs, std::vector<TensorShape> &outputs) override;
|
||||||
|
|
||||||
|
/// Number of inputs the tensor operation accepts
|
||||||
|
uint32_t NumInput() override { return 0; }
|
||||||
|
|
||||||
|
private:
|
||||||
|
int8_t axis_;
|
||||||
|
std::shared_ptr<Tensor> prepend_;
|
||||||
|
std::shared_ptr<Tensor> append_;
|
||||||
|
};
|
||||||
|
} // namespace dataset
|
||||||
|
} // namespace mindspore
|
||||||
|
|
||||||
|
#endif // MINDSPORE_CONCATENATE_OP_H
|
||||||
@ -0,0 +1,66 @@
|
|||||||
|
/**
|
||||||
|
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
#include "common/common.h"
|
||||||
|
#include "dataset/kernels/data/concatenate_op.h"
|
||||||
|
#include "utils/log_adapter.h"
|
||||||
|
|
||||||
|
using namespace mindspore::dataset;
|
||||||
|
using mindspore::LogStream;
|
||||||
|
using mindspore::ExceptionType::NoExceptionType;
|
||||||
|
using mindspore::MsLogLevel::INFO;
|
||||||
|
|
||||||
|
class MindDataTestConcatenateOp : public UT::Common {
|
||||||
|
protected:
|
||||||
|
MindDataTestConcatenateOp() {}
|
||||||
|
};
|
||||||
|
|
||||||
|
TEST_F(MindDataTestConcatenateOp, TestOp) {
|
||||||
|
MS_LOG(INFO) << "Doing MindDataTestConcatenate-TestOp.";
|
||||||
|
uint64_t labels[3] = {1, 1, 2};
|
||||||
|
TensorShape shape({3});
|
||||||
|
std::shared_ptr<Tensor> input =
|
||||||
|
std::make_shared<Tensor>(shape, DataType(DataType::DE_UINT64), reinterpret_cast<unsigned char *>(labels));
|
||||||
|
|
||||||
|
uint64_t append_labels[3] = {4, 4, 4};
|
||||||
|
std::shared_ptr<Tensor> append =
|
||||||
|
std::make_shared<Tensor>(shape, DataType(DataType::DE_UINT64), reinterpret_cast<unsigned char *>(append_labels));
|
||||||
|
|
||||||
|
std::shared_ptr<Tensor> output;
|
||||||
|
std::unique_ptr<ConcatenateOp> op(new ConcatenateOp(0, nullptr, append));
|
||||||
|
TensorRow in;
|
||||||
|
in.push_back(input);
|
||||||
|
TensorRow out_row;
|
||||||
|
Status s = op->Compute(in, &out_row);
|
||||||
|
uint64_t out[6] = {1, 1, 2, 4, 4, 4};
|
||||||
|
|
||||||
|
std::shared_ptr<Tensor> expected =
|
||||||
|
std::make_shared<Tensor>(TensorShape{6}, DataType(DataType::DE_UINT64), reinterpret_cast<unsigned char *>(out));
|
||||||
|
output = out_row[0];
|
||||||
|
EXPECT_TRUE(s.IsOk());
|
||||||
|
ASSERT_TRUE(output->shape() == expected->shape());
|
||||||
|
ASSERT_TRUE(output->type() == expected->type());
|
||||||
|
MS_LOG(DEBUG) << *output << std::endl;
|
||||||
|
MS_LOG(DEBUG) << *expected << std::endl;
|
||||||
|
|
||||||
|
ASSERT_TRUE(*output == *expected);
|
||||||
|
|
||||||
|
// std::vector<TensorShape> inputs = {TensorShape({3})};
|
||||||
|
// std::vector<TensorShape> outputs = {};
|
||||||
|
// s = op->OutputShape(inputs, outputs);
|
||||||
|
// EXPECT_TRUE(s.IsOk());
|
||||||
|
// ASSERT_TRUE(outputs[0] == TensorShape{6});
|
||||||
|
// MS_LOG(INFO) << "MindDataTestConcatenateOp-TestOp end.";
|
||||||
|
}
|
||||||
@ -0,0 +1,175 @@
|
|||||||
|
# Copyright 2020 Huawei Technologies Co., Ltd
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
# ==============================================================================
|
||||||
|
"""
|
||||||
|
Testing concatenate op
|
||||||
|
"""
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
import mindspore.dataset as ds
|
||||||
|
import mindspore.dataset.transforms.c_transforms as data_trans
|
||||||
|
|
||||||
|
|
||||||
|
def test_concatenate_op_all():
|
||||||
|
def gen():
|
||||||
|
yield (np.array([5., 6., 7., 8.], dtype=np.float),)
|
||||||
|
|
||||||
|
prepend_tensor = np.array([1.4, 2., 3., 4., 4.5], dtype=np.float)
|
||||||
|
append_tensor = np.array([9., 10.3, 11., 12.], dtype=np.float)
|
||||||
|
data = ds.GeneratorDataset(gen, column_names=["col"])
|
||||||
|
concatenate_op = data_trans.Concatenate(0, prepend_tensor, append_tensor)
|
||||||
|
data = data.map(input_columns=["col"], operations=concatenate_op)
|
||||||
|
expected = np.array([1.4, 2., 3., 4., 4.5, 5., 6., 7., 8., 9., 10.3,
|
||||||
|
11., 12.])
|
||||||
|
for data_row in data:
|
||||||
|
np.testing.assert_array_equal(data_row[0], expected)
|
||||||
|
|
||||||
|
|
||||||
|
def test_concatenate_op_none():
|
||||||
|
def gen():
|
||||||
|
yield (np.array([5., 6., 7., 8.], dtype=np.float),)
|
||||||
|
|
||||||
|
data = ds.GeneratorDataset(gen, column_names=["col"])
|
||||||
|
concatenate_op = data_trans.Concatenate()
|
||||||
|
|
||||||
|
data = data.map(input_columns=["col"], operations=concatenate_op)
|
||||||
|
for data_row in data:
|
||||||
|
np.testing.assert_array_equal(data_row[0], np.array([5., 6., 7., 8.], dtype=np.float))
|
||||||
|
|
||||||
|
|
||||||
|
def test_concatenate_op_string():
|
||||||
|
def gen():
|
||||||
|
yield (np.array(["ss", "ad"], dtype='S'),)
|
||||||
|
|
||||||
|
prepend_tensor = np.array(["dw", "df"], dtype='S')
|
||||||
|
append_tensor = np.array(["dwsdf", "df"], dtype='S')
|
||||||
|
data = ds.GeneratorDataset(gen, column_names=["col"])
|
||||||
|
concatenate_op = data_trans.Concatenate(0, prepend_tensor, append_tensor)
|
||||||
|
|
||||||
|
data = data.map(input_columns=["col"], operations=concatenate_op)
|
||||||
|
expected = np.array(["dw", "df", "ss", "ad", "dwsdf", "df"], dtype='S')
|
||||||
|
for data_row in data:
|
||||||
|
np.testing.assert_array_equal(data_row[0], expected)
|
||||||
|
|
||||||
|
|
||||||
|
def test_concatenate_op_multi_input_string():
|
||||||
|
prepend_tensor = np.array(["dw", "df"], dtype='S')
|
||||||
|
append_tensor = np.array(["dwsdf", "df"], dtype='S')
|
||||||
|
|
||||||
|
data = ([["1", "2", "d"]], [["3", "4", "e"]])
|
||||||
|
data = ds.NumpySlicesDataset(data, column_names=["col1", "col2"])
|
||||||
|
|
||||||
|
concatenate_op = data_trans.Concatenate(0, prepend=prepend_tensor, append=append_tensor)
|
||||||
|
|
||||||
|
data = data.map(input_columns=["col1", "col2"], columns_order=["out1"], output_columns=["out1"],
|
||||||
|
operations=concatenate_op)
|
||||||
|
expected = np.array(["dw", "df", "1", "2", "d", "3", "4", "e", "dwsdf", "df"], dtype='S')
|
||||||
|
for data_row in data:
|
||||||
|
np.testing.assert_array_equal(data_row[0], expected)
|
||||||
|
|
||||||
|
|
||||||
|
def test_concatenate_op_multi_input_numeric():
|
||||||
|
prepend_tensor = np.array([3, 5])
|
||||||
|
|
||||||
|
data = ([[1, 2]], [[3, 4]])
|
||||||
|
data = ds.NumpySlicesDataset(data, column_names=["col1", "col2"])
|
||||||
|
|
||||||
|
concatenate_op = data_trans.Concatenate(0, prepend=prepend_tensor)
|
||||||
|
|
||||||
|
data = data.map(input_columns=["col1", "col2"], columns_order=["out1"], output_columns=["out1"],
|
||||||
|
operations=concatenate_op)
|
||||||
|
expected = np.array([3, 5, 1, 2, 3, 4])
|
||||||
|
for data_row in data:
|
||||||
|
np.testing.assert_array_equal(data_row[0], expected)
|
||||||
|
|
||||||
|
|
||||||
|
def test_concatenate_op_type_mismatch():
|
||||||
|
def gen():
|
||||||
|
yield (np.array([3, 4], dtype=np.float),)
|
||||||
|
|
||||||
|
prepend_tensor = np.array(["ss", "ad"], dtype='S')
|
||||||
|
data = ds.GeneratorDataset(gen, column_names=["col"])
|
||||||
|
concatenate_op = data_trans.Concatenate(0, prepend_tensor)
|
||||||
|
|
||||||
|
data = data.map(input_columns=["col"], operations=concatenate_op)
|
||||||
|
with pytest.raises(RuntimeError) as error_info:
|
||||||
|
for _ in data:
|
||||||
|
pass
|
||||||
|
assert "Tensor types do not match" in repr(error_info.value)
|
||||||
|
|
||||||
|
|
||||||
|
def test_concatenate_op_type_mismatch2():
|
||||||
|
def gen():
|
||||||
|
yield (np.array(["ss", "ad"], dtype='S'),)
|
||||||
|
|
||||||
|
prepend_tensor = np.array([3, 5], dtype=np.float)
|
||||||
|
data = ds.GeneratorDataset(gen, column_names=["col"])
|
||||||
|
concatenate_op = data_trans.Concatenate(0, prepend_tensor)
|
||||||
|
|
||||||
|
data = data.map(input_columns=["col"], operations=concatenate_op)
|
||||||
|
with pytest.raises(RuntimeError) as error_info:
|
||||||
|
for _ in data:
|
||||||
|
pass
|
||||||
|
assert "Tensor types do not match" in repr(error_info.value)
|
||||||
|
|
||||||
|
|
||||||
|
def test_concatenate_op_incorrect_dim():
|
||||||
|
def gen():
|
||||||
|
yield (np.array([["ss", "ad"], ["ss", "ad"]], dtype='S'),)
|
||||||
|
|
||||||
|
prepend_tensor = np.array([3, 5], dtype=np.float)
|
||||||
|
concatenate_op = data_trans.Concatenate(0, prepend_tensor)
|
||||||
|
data = ds.GeneratorDataset(gen, column_names=["col"])
|
||||||
|
|
||||||
|
data = data.map(input_columns=["col"], operations=concatenate_op)
|
||||||
|
with pytest.raises(RuntimeError) as error_info:
|
||||||
|
for _ in data:
|
||||||
|
pass
|
||||||
|
assert "Only 1D tensors supported" in repr(error_info.value)
|
||||||
|
|
||||||
|
|
||||||
|
def test_concatenate_op_wrong_axis():
|
||||||
|
with pytest.raises(ValueError) as error_info:
|
||||||
|
data_trans.Concatenate(2)
|
||||||
|
assert "only 1D concatenation supported." in repr(error_info.value)
|
||||||
|
|
||||||
|
|
||||||
|
def test_concatenate_op_incorrect_input_dim():
|
||||||
|
def gen():
|
||||||
|
yield (np.array(["ss", "ad"], dtype='S'),)
|
||||||
|
|
||||||
|
prepend_tensor = np.array([["ss", "ad"], ["ss", "ad"]], dtype='S')
|
||||||
|
data = ds.GeneratorDataset(gen, column_names=["col"])
|
||||||
|
concatenate_op = data_trans.Concatenate(0, prepend_tensor)
|
||||||
|
|
||||||
|
data = data.map(input_columns=["col"], operations=concatenate_op)
|
||||||
|
with pytest.raises(RuntimeError) as error_info:
|
||||||
|
for _ in data:
|
||||||
|
pass
|
||||||
|
assert "Only 1D tensors supported" in repr(error_info.value)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
test_concatenate_op_all()
|
||||||
|
test_concatenate_op_none()
|
||||||
|
test_concatenate_op_string()
|
||||||
|
test_concatenate_op_type_mismatch()
|
||||||
|
test_concatenate_op_type_mismatch2()
|
||||||
|
test_concatenate_op_incorrect_dim()
|
||||||
|
test_concatenate_op_incorrect_input_dim()
|
||||||
|
test_concatenate_op_multi_input_numeric()
|
||||||
|
test_concatenate_op_multi_input_string()
|
||||||
|
test_concatenate_op_wrong_axis()
|
||||||
Loading…
Reference in new issue