!1807 Implemented Ngram TensorOp for dataset
Merge pull request !1807 from ZiruiWu/ngram_devpull/1807/MERGE
mindspore-ci-bot
5 years ago
committed by
Gitee
commit
5c21616293
@ -0,0 +1,93 @@
|
||||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "dataset/text/kernels/ngram_op.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace mindspore {
|
||||
namespace dataset {
|
||||
|
||||
NgramOp::NgramOp(const std::vector<int32_t> &ngrams, int32_t l_len, int32_t r_len, const std::string &l_pad,
|
||||
const std::string &r_pad, const std::string &separator)
|
||||
: ngrams_(ngrams),
|
||||
l_len_(l_len),
|
||||
r_len_(r_len),
|
||||
l_pad_with_sp_(l_pad + separator),
|
||||
r_pad_with_sp_(r_pad + separator),
|
||||
separator_(separator) {}
|
||||
|
||||
Status NgramOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(input->type() == DataType::DE_STRING && input->Rank() == 1, "Not a 1-D str Tensor");
|
||||
std::vector<int32_t> offsets; // offsets for each str
|
||||
std::vector<std::string> res; // holds the result of ngrams
|
||||
std::string str_buffer; // concat all pad tokens with string interleaved with separators
|
||||
res.reserve(input->shape().NumOfElements()); // this should be more than enough
|
||||
offsets.reserve(1 + l_len_ + r_len_ + input->shape().NumOfElements());
|
||||
str_buffer.reserve(l_pad_with_sp_.size() * l_len_ + r_pad_with_sp_.size() * r_len_ + input->SizeInBytes());
|
||||
offsets.push_back(str_buffer.size()); // insert 0 as the starting pos
|
||||
for (int i = 0; i < l_len_; i++) offsets.push_back((str_buffer += l_pad_with_sp_).size());
|
||||
|
||||
for (auto itr = input->begin<std::string_view>(); itr != input->end<std::string_view>(); itr++) {
|
||||
str_buffer += (*itr);
|
||||
str_buffer += separator_;
|
||||
offsets.push_back(str_buffer.size());
|
||||
}
|
||||
|
||||
for (int i = 0; i < r_len_; i++) offsets.push_back((str_buffer += r_pad_with_sp_).size());
|
||||
|
||||
for (auto n : ngrams_) {
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(n > 0, "n gram needs to be a positive number.\n");
|
||||
int32_t start_ind = l_len_ - std::min(l_len_, n - 1);
|
||||
int32_t end_ind = offsets.size() - r_len_ + std::min(r_len_, n - 1);
|
||||
if (end_ind - start_ind < n) {
|
||||
res.emplace_back(std::string()); // push back empty string
|
||||
} else {
|
||||
for (int i = start_ind; i < end_ind - n; i++) {
|
||||
res.emplace_back(str_buffer.substr(offsets[i], offsets[i + n] - offsets[i] - separator_.size()));
|
||||
}
|
||||
}
|
||||
}
|
||||
RETURN_IF_NOT_OK(Tensor::CreateTensor(output, res, TensorShape({static_cast<dsize_t>(res.size())})));
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
void NgramOp::Print(std::ostream &out) const {
|
||||
out << "NgramOp: "
|
||||
<< "left pad width: " << l_len_ << " left pad token with separator: " << l_pad_with_sp_ << "\n"
|
||||
<< "right pad width: " << r_len_ << " right pad token with separator: " << r_pad_with_sp_ << "\n"
|
||||
<< "separator: " << separator_ << "\n";
|
||||
}
|
||||
|
||||
Status NgramOp::OutputShape(const std::vector<TensorShape> &inputs, std::vector<TensorShape> &outputs) {
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(inputs.size() == NumInput(), "incorrect num of inputs\n");
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(inputs[0].Rank() == 1, "ngram only works with 1-dim data\n");
|
||||
dsize_t num_elements = ngrams_.size();
|
||||
for (int32_t n : ngrams_) {
|
||||
// here since rank == 1, NumOfElements == shape[0]. add padding length to string
|
||||
int32_t len_with_padding = inputs[0].NumOfElements() + std::min(n - 1, l_len_) + std::min(n - 1, r_len_);
|
||||
// if len_with_padding - n < 0, this would return an empty string
|
||||
num_elements += std::max(len_with_padding - n, 0);
|
||||
}
|
||||
outputs.emplace_back(TensorShape({num_elements}));
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(outputs.size() == NumOutput(), "incorrect num of outputs\n");
|
||||
return Status::OK();
|
||||
}
|
||||
} // namespace dataset
|
||||
} // namespace mindspore
|
||||
@ -0,0 +1,74 @@
|
||||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef DATASET_TEXT_KERNELS_NGRAM_OP_H_
|
||||
#define DATASET_TEXT_KERNELS_NGRAM_OP_H_
|
||||
|
||||
#include <string>
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
#include "dataset/core/tensor.h"
|
||||
#include "dataset/kernels/tensor_op.h"
|
||||
#include "dataset/util/status.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace dataset {
|
||||
namespace py = pybind11;
|
||||
|
||||
class NgramOp : public TensorOp {
|
||||
public:
|
||||
// Constructor of Ngram model
|
||||
// @param const std::vector<int32_t> &ngrams
|
||||
// @param int32_tl_len - padding length on the left
|
||||
// @param int32_t r_len - padding length on the right
|
||||
// @param const std::string &l_pad - padding token on the left
|
||||
// @param const std::string &r_pad - padding token on the right
|
||||
// @param const std::string &separator - use to join strings
|
||||
NgramOp(const std::vector<int32_t> &ngrams, int32_t l_len, int32_t r_len, const std::string &l_pad,
|
||||
const std::string &r_pad, const std::string &separator);
|
||||
|
||||
// perform ngram model on each tensor
|
||||
// @param const std::shared_ptr<Tensor> &input
|
||||
// @param std::shared_ptr<Tensor> *output
|
||||
// @return error code
|
||||
Status Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) override;
|
||||
|
||||
// destructor
|
||||
~NgramOp() override = default;
|
||||
|
||||
// @param std::vector<TensorShape> &inputs - shape of input tensors
|
||||
// @param std::vector<TensorShape> &outputs - shape of output tensors
|
||||
// @return error code
|
||||
Status OutputShape(const std::vector<TensorShape> &inputs, std::vector<TensorShape> &outputs) override;
|
||||
|
||||
// print arg for debugging
|
||||
// @param std::ostream &out
|
||||
void Print(std::ostream &out) const override;
|
||||
|
||||
private:
|
||||
std::vector<int32_t> ngrams_; // list of n grams
|
||||
int32_t l_len_; // left padding length
|
||||
int32_t r_len_; // right padding length
|
||||
std::string l_pad_with_sp_; // left padding appended with separator
|
||||
std::string r_pad_with_sp_; // right padding appended with separator
|
||||
std::string separator_; // separator
|
||||
};
|
||||
|
||||
} // namespace dataset
|
||||
} // namespace mindspore
|
||||
|
||||
#endif // DATASET_TEXT_KERNELS_NGRAM_OP_H_
|
||||
@ -0,0 +1,115 @@
|
||||
# Copyright 2020 Huawei Technologies Co., Ltd
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""
|
||||
Testing NgramOP in DE
|
||||
"""
|
||||
import mindspore.dataset as ds
|
||||
import mindspore.dataset.text as nlp
|
||||
import numpy as np
|
||||
|
||||
|
||||
def test_multiple_ngrams():
|
||||
""" test n-gram where n is a list of integers"""
|
||||
plates_mottos = ["WildRose Country", "Canada's Ocean Playground", "Land of Living Skies"]
|
||||
n_gram_mottos = []
|
||||
n_gram_mottos.append(
|
||||
['WildRose', 'Country', '_ WildRose', 'WildRose Country', 'Country _', '_ _ WildRose', '_ WildRose Country',
|
||||
'WildRose Country _', 'Country _ _'])
|
||||
n_gram_mottos.append(
|
||||
["Canada's", 'Ocean', 'Playground', "_ Canada's", "Canada's Ocean", 'Ocean Playground', 'Playground _',
|
||||
"_ _ Canada's", "_ Canada's Ocean", "Canada's Ocean Playground", 'Ocean Playground _', 'Playground _ _'])
|
||||
n_gram_mottos.append(
|
||||
['Land', 'of', 'Living', 'Skies', '_ Land', 'Land of', 'of Living', 'Living Skies', 'Skies _', '_ _ Land',
|
||||
'_ Land of', 'Land of Living', 'of Living Skies', 'Living Skies _', 'Skies _ _'])
|
||||
|
||||
def gen(texts):
|
||||
for line in texts:
|
||||
yield (np.array(line.split(" "), dtype='S'),)
|
||||
|
||||
dataset = ds.GeneratorDataset(gen(plates_mottos), column_names=["text"])
|
||||
dataset = dataset.map(input_columns=["text"], operations=nlp.Ngram([1, 2, 3], ("_", 2), ("_", 2), " "))
|
||||
|
||||
i = 0
|
||||
for data in dataset.create_dict_iterator():
|
||||
assert [d.decode("utf8") for d in data["text"]] == n_gram_mottos[i]
|
||||
i += 1
|
||||
|
||||
|
||||
def test_simple_ngram():
|
||||
""" test simple gram with only one n value"""
|
||||
plates_mottos = ["Friendly Manitoba", "Yours to Discover", "Land of Living Skies",
|
||||
"Birthplace of the Confederation"]
|
||||
n_gram_mottos = [[]]
|
||||
n_gram_mottos.append(["Yours to Discover"])
|
||||
n_gram_mottos.append(['Land of Living', 'of Living Skies'])
|
||||
n_gram_mottos.append(['Birthplace of the', 'of the Confederation'])
|
||||
|
||||
def gen(texts):
|
||||
for line in texts:
|
||||
yield (np.array(line.split(" "), dtype='S'),)
|
||||
|
||||
dataset = ds.GeneratorDataset(gen(plates_mottos), column_names=["text"])
|
||||
dataset = dataset.map(input_columns=["text"], operations=nlp.Ngram(3, separator=None))
|
||||
|
||||
i = 0
|
||||
for data in dataset.create_dict_iterator():
|
||||
assert [d.decode("utf8") for d in data["text"]] == n_gram_mottos[i], i
|
||||
i += 1
|
||||
|
||||
|
||||
def test_corner_cases():
|
||||
""" testing various corner cases and exceptions"""
|
||||
|
||||
def test_config(input_line, output_line, n, l_pad=None, r_pad=None, sep=None):
|
||||
def gen(text):
|
||||
yield (np.array(text.split(" "), dtype='S'),)
|
||||
|
||||
dataset = ds.GeneratorDataset(gen(input_line), column_names=["text"])
|
||||
dataset = dataset.map(input_columns=["text"], operations=nlp.Ngram(n, l_pad, r_pad, separator=sep))
|
||||
for data in dataset.create_dict_iterator():
|
||||
assert [d.decode("utf8") for d in data["text"]] == output_line, output_line
|
||||
|
||||
# test empty separator
|
||||
test_config("Beautiful British Columbia", ['BeautifulBritish', 'BritishColumbia'], 2, sep="")
|
||||
# test separator with longer length
|
||||
test_config("Beautiful British Columbia", ['Beautiful^-^British^-^Columbia'], 3, sep="^-^")
|
||||
# test left pad != right pad
|
||||
test_config("Lone Star", ['The Lone Star State'], 4, ("The", 1), ("State", 1))
|
||||
# test invalid n
|
||||
try:
|
||||
test_config("Yours to Discover", "", [0, [1]])
|
||||
except Exception as e:
|
||||
assert "ngram needs to be a positive number" in str(e)
|
||||
# test empty n
|
||||
try:
|
||||
test_config("Yours to Discover", "", [])
|
||||
except Exception as e:
|
||||
assert "n needs to be a non-empty list" in str(e)
|
||||
# test invalid pad
|
||||
try:
|
||||
test_config("Yours to Discover", "", [1], ("str", -1))
|
||||
except Exception as e:
|
||||
assert "padding width need to be positive numbers" in str(e)
|
||||
# test invalid pad
|
||||
try:
|
||||
test_config("Yours to Discover", "", [1], ("str", "rts"))
|
||||
except Exception as e:
|
||||
assert "pad needs to be a tuple of (str, int)" in str(e)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
test_multiple_ngrams()
|
||||
test_simple_ngram()
|
||||
test_corner_cases()
|
||||
Loading…
Reference in new issue