hesham
5 years ago
parent
cc80dca7ed
commit
e2012a1de9
@ -0,0 +1,48 @@
|
||||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
#include "dataset/kernels/data/slice_op.h"
|
||||
|
||||
#include "dataset/core/tensor.h"
|
||||
#include "dataset/kernels/data/data_utils.h"
|
||||
#include "dataset/kernels/tensor_op.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace dataset {
|
||||
Status SliceOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
|
||||
IO_CHECK(input, output);
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(input->shape().Rank() == 1, "SliceOp supports 1D Tensors only for now.");
|
||||
|
||||
// if `all` flag is true, output is just the input.
|
||||
if (all_) {
|
||||
*output = input;
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
// if slice object was provided, indices should be empty. Generate indices from the slice object.
|
||||
if (slice_.valid() && indices_.empty()) {
|
||||
dsize_t len = input->shape()[0];
|
||||
indices_ = slice_.Indices(len);
|
||||
return input->Slice(output, indices_);
|
||||
}
|
||||
|
||||
// if indices are not empty, slices should be invalid, use indices_ to slice
|
||||
if (!indices_.empty() && !slice_.valid()) {
|
||||
return input->Slice(output, indices_);
|
||||
}
|
||||
RETURN_STATUS_UNEXPECTED("The indexing parameters are invalid");
|
||||
}
|
||||
} // namespace dataset
|
||||
} // namespace mindspore
|
||||
@ -0,0 +1,83 @@
|
||||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
#ifndef DATASET_KERNELS_DATA_SLICE_OP_H_
|
||||
#define DATASET_KERNELS_DATA_SLICE_OP_H_
|
||||
|
||||
#include <algorithm>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "dataset/core/tensor.h"
|
||||
#include "dataset/kernels/tensor_op.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace dataset {
|
||||
class Slice {
|
||||
public:
|
||||
Slice() : start_(0), stop_(0), step_(0) {}
|
||||
Slice(dsize_t start, dsize_t stop, dsize_t step) : start_(start), stop_(stop), step_(step) {}
|
||||
Slice(dsize_t start, dsize_t stop) : start_(start), stop_(stop), step_(1) {}
|
||||
explicit Slice(dsize_t stop) : start_(0), stop_(stop), step_(1) {}
|
||||
|
||||
std::vector<dsize_t> Indices(dsize_t length) {
|
||||
std::vector<dsize_t> indices;
|
||||
dsize_t index = std::min(Tensor::handleNeg(start_, length), length);
|
||||
dsize_t end_index = std::min(Tensor::handleNeg(stop_, length), length);
|
||||
if (step_ > 0) {
|
||||
for (; index < end_index; index += step_) {
|
||||
indices.push_back(index);
|
||||
}
|
||||
} else {
|
||||
for (; index > end_index; index += step_) {
|
||||
indices.push_back(index);
|
||||
}
|
||||
}
|
||||
return indices;
|
||||
}
|
||||
|
||||
bool valid() { return !(start_ == 0 && stop_ == 0 && step_ == 0); }
|
||||
|
||||
dsize_t start_;
|
||||
dsize_t stop_;
|
||||
dsize_t step_;
|
||||
};
|
||||
|
||||
class SliceOp : public TensorOp {
|
||||
public:
|
||||
explicit SliceOp(std::vector<dsize_t> indices) : indices_(std::move(indices)) {}
|
||||
explicit SliceOp(Slice slice) : slice_(slice) {}
|
||||
explicit SliceOp(bool all) : all_(all) {}
|
||||
|
||||
~SliceOp() override = default;
|
||||
|
||||
void Print(std::ostream &out) const override { out << "SliceOp"; }
|
||||
|
||||
Status Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) override;
|
||||
|
||||
private:
|
||||
// only on of the following will be valid
|
||||
// given indices to slice the Tensor. Empty vector if invalid.
|
||||
std::vector<dsize_t> indices_;
|
||||
// Slice object. All start, stop and step are 0 if invalid.
|
||||
Slice slice_;
|
||||
// Flag to read all indcies in the dim.
|
||||
bool all_ = false;
|
||||
};
|
||||
} // namespace dataset
|
||||
} // namespace mindspore
|
||||
#endif // DATASET_KERNELS_DATA_ONE_HOT_OP_H_
|
||||
@ -0,0 +1,211 @@
|
||||
# Copyright 2019 Huawei Technologies Co., Ltd
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""
|
||||
Testing TypeCast op in DE
|
||||
"""
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
import mindspore.dataset as ds
|
||||
import mindspore.dataset.transforms.c_transforms as ops
|
||||
|
||||
|
||||
def slice_compare(array, indexing):
|
||||
data = ds.NumpySlicesDataset([array])
|
||||
array = np.array(array)
|
||||
data = data.map(operations=ops.Slice(indexing))
|
||||
for d in data:
|
||||
if indexing is None:
|
||||
array = array[:]
|
||||
else:
|
||||
array = array[indexing]
|
||||
np.testing.assert_array_equal(array, d[0])
|
||||
|
||||
|
||||
def test_slice_all():
|
||||
slice_compare([1, 2, 3, 4, 5], None)
|
||||
slice_compare([1, 2, 3, 4, 5], ...)
|
||||
|
||||
|
||||
def test_slice_single_index():
|
||||
slice_compare([1, 2, 3, 4, 5], 0)
|
||||
slice_compare([1, 2, 3, 4, 5], 4)
|
||||
slice_compare([1, 2, 3, 4, 5], 2)
|
||||
slice_compare([1, 2, 3, 4, 5], -1)
|
||||
slice_compare([1, 2, 3, 4, 5], -5)
|
||||
slice_compare([1, 2, 3, 4, 5], -3)
|
||||
|
||||
|
||||
def test_slice_list_index():
|
||||
slice_compare([1, 2, 3, 4, 5], [0, 1, 4])
|
||||
slice_compare([1, 2, 3, 4, 5], [4, 1, 0])
|
||||
slice_compare([1, 2, 3, 4, 5], [-1, 1, 0])
|
||||
slice_compare([1, 2, 3, 4, 5], [-1, -4, -2])
|
||||
slice_compare([1, 2, 3, 4, 5], [3, 3, 3])
|
||||
slice_compare([1, 2, 3, 4, 5], [1, 1, 1, 1, 1])
|
||||
|
||||
|
||||
def test_slice_slice_obj_2s():
|
||||
slice_compare([1, 2, 3, 4, 5], slice(0, 2))
|
||||
slice_compare([1, 2, 3, 4, 5], slice(2, 4))
|
||||
slice_compare([1, 2, 3, 4, 5], slice(4, 10))
|
||||
|
||||
|
||||
def test_slice_slice_obj_1s():
|
||||
slice_compare([1, 2, 3, 4, 5], slice(1))
|
||||
slice_compare([1, 2, 3, 4, 5], slice(4))
|
||||
slice_compare([1, 2, 3, 4, 5], slice(10))
|
||||
|
||||
|
||||
def test_slice_slice_obj_3s():
|
||||
slice_compare([1, 2, 3, 4, 5], slice(0, 2, 1))
|
||||
slice_compare([1, 2, 3, 4, 5], slice(0, 4, 1))
|
||||
slice_compare([1, 2, 3, 4, 5], slice(0, 10, 1))
|
||||
slice_compare([1, 2, 3, 4, 5], slice(0, 5, 2))
|
||||
slice_compare([1, 2, 3, 4, 5], slice(0, 2, 2))
|
||||
slice_compare([1, 2, 3, 4, 5], slice(0, 1, 2))
|
||||
slice_compare([1, 2, 3, 4, 5], slice(4, 5, 1))
|
||||
slice_compare([1, 2, 3, 4, 5], slice(2, 5, 3))
|
||||
|
||||
|
||||
def test_slice_slice_obj_3s_double():
|
||||
slice_compare([1., 2., 3., 4., 5.], slice(0, 2, 1))
|
||||
slice_compare([1., 2., 3., 4., 5.], slice(0, 4, 1))
|
||||
slice_compare([1., 2., 3., 4., 5.], slice(0, 10, 1))
|
||||
slice_compare([1., 2., 3., 4., 5.], slice(0, 5, 2))
|
||||
slice_compare([1., 2., 3., 4., 5.], slice(0, 2, 2))
|
||||
slice_compare([1., 2., 3., 4., 5.], slice(0, 1, 2))
|
||||
slice_compare([1., 2., 3., 4., 5.], slice(4, 5, 1))
|
||||
slice_compare([1., 2., 3., 4., 5.], slice(2, 5, 3))
|
||||
|
||||
|
||||
def test_slice_slice_obj_neg():
|
||||
slice_compare([1, 2, 3, 4, 5], slice(-1, -5, -1))
|
||||
slice_compare([1, 2, 3, 4, 5], slice(-1))
|
||||
slice_compare([1, 2, 3, 4, 5], slice(-2))
|
||||
slice_compare([1, 2, 3, 4, 5], slice(-1, -5, -2))
|
||||
slice_compare([1, 2, 3, 4, 5], slice(-5, -1, 2))
|
||||
slice_compare([1, 2, 3, 4, 5], slice(-5, -1))
|
||||
|
||||
|
||||
def test_slice_exceptions():
|
||||
with pytest.raises(RuntimeError) as info:
|
||||
slice_compare([1, 2, 3, 4, 5], 5)
|
||||
assert "Index 5 is out of bounds [0,5)" in str(info.value)
|
||||
|
||||
with pytest.raises(RuntimeError) as info:
|
||||
slice_compare([1, 2, 3, 4, 5], slice(0))
|
||||
assert "Indices are empty, generated tensor would be empty." in str(info.value)
|
||||
|
||||
with pytest.raises(RuntimeError) as info:
|
||||
slice_compare([1, 2, 3, 4, 5], slice(5, 10, 1))
|
||||
assert "Indices are empty, generated tensor would be empty." in str(info.value)
|
||||
|
||||
with pytest.raises(RuntimeError) as info:
|
||||
slice_compare([1, 2, 3, 4, 5], slice(-1, -5, 1))
|
||||
assert "Indices are empty, generated tensor would be empty." in str(info.value)
|
||||
|
||||
|
||||
def test_slice_all_str():
|
||||
slice_compare([b"1", b"2", b"3", b"4", b"5"], None)
|
||||
slice_compare([b"1", b"2", b"3", b"4", b"5"], ...)
|
||||
|
||||
|
||||
def test_slice_single_index_str():
|
||||
slice_compare([b"1", b"2", b"3", b"4", b"5"], 0)
|
||||
slice_compare([b"1", b"2", b"3", b"4", b"5"], 4)
|
||||
slice_compare([b"1", b"2", b"3", b"4", b"5"], 2)
|
||||
slice_compare([b"1", b"2", b"3", b"4", b"5"], -1)
|
||||
slice_compare([b"1", b"2", b"3", b"4", b"5"], -5)
|
||||
slice_compare([b"1", b"2", b"3", b"4", b"5"], -3)
|
||||
|
||||
|
||||
def test_slice_list_index_str():
|
||||
slice_compare([b"1", b"2", b"3", b"4", b"5"], [0, 1, 4])
|
||||
slice_compare([b"1", b"2", b"3", b"4", b"5"], [4, 1, 0])
|
||||
slice_compare([b"1", b"2", b"3", b"4", b"5"], [-1, 1, 0])
|
||||
slice_compare([b"1", b"2", b"3", b"4", b"5"], [-1, -4, -2])
|
||||
slice_compare([b"1", b"2", b"3", b"4", b"5"], [3, 3, 3])
|
||||
slice_compare([b"1", b"2", b"3", b"4", b"5"], [1, 1, 1, 1, 1])
|
||||
|
||||
|
||||
def test_slice_slice_obj_2s_str():
|
||||
slice_compare([b"1", b"2", b"3", b"4", b"5"], slice(0, 2))
|
||||
slice_compare([b"1", b"2", b"3", b"4", b"5"], slice(2, 4))
|
||||
slice_compare([b"1", b"2", b"3", b"4", b"5"], slice(4, 10))
|
||||
|
||||
|
||||
def test_slice_slice_obj_1s_str():
|
||||
slice_compare([b"1", b"2", b"3", b"4", b"5"], slice(1))
|
||||
slice_compare([b"1", b"2", b"3", b"4", b"5"], slice(4))
|
||||
slice_compare([b"1", b"2", b"3", b"4", b"5"], slice(10))
|
||||
|
||||
|
||||
def test_slice_slice_obj_3s_str():
|
||||
slice_compare([b"1", b"2", b"3", b"4", b"5"], slice(0, 2, 1))
|
||||
slice_compare([b"1", b"2", b"3", b"4", b"5"], slice(0, 4, 1))
|
||||
slice_compare([b"1", b"2", b"3", b"4", b"5"], slice(0, 10, 1))
|
||||
slice_compare([b"1", b"2", b"3", b"4", b"5"], slice(0, 5, 2))
|
||||
slice_compare([b"1", b"2", b"3", b"4", b"5"], slice(0, 2, 2))
|
||||
slice_compare([b"1", b"2", b"3", b"4", b"5"], slice(0, 1, 2))
|
||||
slice_compare([b"1", b"2", b"3", b"4", b"5"], slice(4, 5, 1))
|
||||
slice_compare([b"1", b"2", b"3", b"4", b"5"], slice(2, 5, 3))
|
||||
|
||||
|
||||
def test_slice_slice_obj_neg_str():
|
||||
slice_compare([b"1", b"2", b"3", b"4", b"5"], slice(-1, -5, -1))
|
||||
slice_compare([b"1", b"2", b"3", b"4", b"5"], slice(-1))
|
||||
slice_compare([b"1", b"2", b"3", b"4", b"5"], slice(-2))
|
||||
slice_compare([b"1", b"2", b"3", b"4", b"5"], slice(-1, -5, -2))
|
||||
slice_compare([b"1", b"2", b"3", b"4", b"5"], slice(-5, -1, 2))
|
||||
slice_compare([b"1", b"2", b"3", b"4", b"5"], slice(-5, -1))
|
||||
|
||||
|
||||
def test_slice_exceptions_str():
|
||||
with pytest.raises(RuntimeError) as info:
|
||||
slice_compare([b"1", b"2", b"3", b"4", b"5"], 5)
|
||||
assert "Index 5 is out of bounds [0,5)" in str(info.value)
|
||||
|
||||
with pytest.raises(RuntimeError) as info:
|
||||
slice_compare([b"1", b"2", b"3", b"4", b"5"], slice(0))
|
||||
assert "Indices are empty, generated tensor would be empty." in str(info.value)
|
||||
|
||||
with pytest.raises(RuntimeError) as info:
|
||||
slice_compare([b"1", b"2", b"3", b"4", b"5"], slice(5, 10, 1))
|
||||
assert "Indices are empty, generated tensor would be empty." in str(info.value)
|
||||
|
||||
with pytest.raises(RuntimeError) as info:
|
||||
slice_compare([b"1", b"2", b"3", b"4", b"5"], slice(-1, -5, 1))
|
||||
assert "Indices are empty, generated tensor would be empty." in str(info.value)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_slice_all()
|
||||
test_slice_single_index()
|
||||
test_slice_list_index()
|
||||
test_slice_slice_obj_3s()
|
||||
test_slice_slice_obj_2s()
|
||||
test_slice_slice_obj_1s()
|
||||
test_slice_slice_obj_neg()
|
||||
test_slice_exceptions()
|
||||
test_slice_slice_obj_3s_double()
|
||||
test_slice_all_str()
|
||||
test_slice_single_index_str()
|
||||
test_slice_list_index_str()
|
||||
test_slice_slice_obj_3s_str()
|
||||
test_slice_slice_obj_2s_str()
|
||||
test_slice_slice_obj_1s_str()
|
||||
test_slice_slice_obj_neg_str()
|
||||
test_slice_exceptions_str()
|
||||
Loading…
Reference in new issue