!4597 [Dataset] C++ API Support for build_vocab
Merge pull request !4597 from luoyang/c-apipull/4597/MERGE
mindspore-ci-bot
5 years ago
committed by
Gitee
commit
89cd465268
@ -0,0 +1,64 @@
|
||||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "minddata/dataset/include/text.h"
|
||||
#include "minddata/dataset/text/kernels/lookup_op.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace dataset {
|
||||
namespace api {
|
||||
namespace text {
|
||||
|
||||
std::shared_ptr<LookupOperation> Lookup(const std::shared_ptr<Vocab> &vocab, const std::string &unknown_token) {
|
||||
auto op = std::make_shared<LookupOperation>(vocab, unknown_token);
|
||||
|
||||
if (!op->ValidateParams()) {
|
||||
return nullptr;
|
||||
}
|
||||
return op;
|
||||
}
|
||||
|
||||
// LookupOperation
|
||||
LookupOperation::LookupOperation(const std::shared_ptr<Vocab> &vocab, const std::string &unknown_token)
|
||||
: vocab_(vocab), unknown_token_(unknown_token), default_id_(Vocab::kNoTokenExists) {}
|
||||
|
||||
bool LookupOperation::ValidateParams() {
|
||||
if (vocab_ == nullptr) {
|
||||
LOG(ERROR) << "Lookup: vocab object type is incorrect or null.";
|
||||
return false;
|
||||
}
|
||||
if (unknown_token_.empty()) {
|
||||
LOG(ERROR) << "Lookup: no unknown token is specified.";
|
||||
return false;
|
||||
} else {
|
||||
default_id_ = vocab_->Lookup(unknown_token_);
|
||||
if (default_id_ == Vocab::kNoTokenExists) {
|
||||
LOG(ERROR) << "Lookup: unknown_token: [" + unknown_token_ + "], does not exist in vocab.";
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
std::shared_ptr<TensorOp> LookupOperation::Build() {
|
||||
std::shared_ptr<LookupOp> tensor_op = std::make_shared<LookupOp>(vocab_, default_id_);
|
||||
return tensor_op;
|
||||
}
|
||||
|
||||
} // namespace text
|
||||
} // namespace api
|
||||
} // namespace dataset
|
||||
} // namespace mindspore
|
||||
@ -0,0 +1,65 @@
|
||||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_API_TEXT_H_
|
||||
#define MINDSPORE_CCSRC_MINDDATA_DATASET_API_TEXT_H_
|
||||
|
||||
#include <vector>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include "minddata/dataset/core/constants.h"
|
||||
#include "minddata/dataset/include/transforms.h"
|
||||
#include "minddata/dataset/text/vocab.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace dataset {
|
||||
namespace api {
|
||||
|
||||
// Transform operations for text
|
||||
namespace text {
|
||||
|
||||
// Text Op classes (in alphabetical order)
|
||||
class LookupOperation;
|
||||
|
||||
/// \brief Lookup operator that looks up a word to an id.
|
||||
/// \param[in] vocab a Vocab object.
|
||||
/// \param[in] unknown_token word to use for lookup if the word being looked up is out of Vocabulary (oov).
|
||||
/// If unknown_token is oov, runtime error will be thrown
|
||||
/// \return Shared pointer to the current TensorOperation.
|
||||
std::shared_ptr<LookupOperation> Lookup(const std::shared_ptr<Vocab> &vocab, const std::string &unknown_token);
|
||||
|
||||
/* ####################################### Derived TensorOperation classes ################################# */
|
||||
|
||||
class LookupOperation : public TensorOperation {
|
||||
public:
|
||||
explicit LookupOperation(const std::shared_ptr<Vocab> &vocab, const std::string &unknown_token);
|
||||
|
||||
~LookupOperation() = default;
|
||||
|
||||
std::shared_ptr<TensorOp> Build() override;
|
||||
|
||||
bool ValidateParams() override;
|
||||
|
||||
private:
|
||||
std::shared_ptr<Vocab> vocab_;
|
||||
std::string unknown_token_;
|
||||
int32_t default_id_;
|
||||
};
|
||||
} // namespace text
|
||||
} // namespace api
|
||||
} // namespace dataset
|
||||
} // namespace mindspore
|
||||
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_API_TEXT_H_
|
||||
@ -0,0 +1,229 @@
|
||||
/**
|
||||
* Copyright 2020 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
#include <string>
|
||||
|
||||
#include "common/common.h"
|
||||
#include "minddata/dataset/include/datasets.h"
|
||||
#include "minddata/dataset/include/status.h"
|
||||
|
||||
using mindspore::dataset::Tensor;
|
||||
using mindspore::dataset::Status;
|
||||
using mindspore::dataset::Vocab;
|
||||
|
||||
class MindDataTestVocab : public UT::DatasetOpTesting {
|
||||
protected:
|
||||
};
|
||||
|
||||
TEST_F(MindDataTestVocab, TestVocabFromUnorderedMap) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestVocab-TestVocabFromUnorderedMap.";
|
||||
// Build a map
|
||||
std::unordered_map<std::string, int32_t> dict;
|
||||
dict["banana"] = 0;
|
||||
dict["apple"] = 1;
|
||||
dict["cat"] = 2;
|
||||
dict["dog"] = 3;
|
||||
|
||||
// Build vocab from map
|
||||
std::shared_ptr<Vocab> vocab = std::make_shared<Vocab>();
|
||||
Status s = Vocab::BuildFromUnorderedMap(dict, &vocab);
|
||||
EXPECT_EQ(s, Status::OK());
|
||||
|
||||
// Look up specified words
|
||||
std::vector<std::string> words = {"apple", "dog", "egg"};
|
||||
std::vector<int32_t> expected = {1, 3, -1};
|
||||
for (uint32_t i = 0; i < words.size(); ++i) {
|
||||
int32_t x = vocab->Lookup(words[i]);
|
||||
EXPECT_EQ(x, expected[i]);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(MindDataTestVocab, TestVocabFromEmptyMap) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestVocab-TestVocabFromEmptyMap.";
|
||||
// Build vocab from empty map
|
||||
std::unordered_map<std::string, int32_t> dict;
|
||||
std::shared_ptr<Vocab> vocab = std::make_shared<Vocab>();
|
||||
Status s = Vocab::BuildFromUnorderedMap(dict, &vocab);
|
||||
EXPECT_EQ(s, Status::OK());
|
||||
|
||||
// Look up specified words
|
||||
// Expect that we will return -1 when word is not in vocab
|
||||
std::vector<std::string> words = {"apple", "dog", "egg"};
|
||||
std::vector<int32_t> expected = {-1, -1, -1};
|
||||
for (uint32_t i = 0; i < words.size(); ++i) {
|
||||
int32_t x = vocab->Lookup(words[i]);
|
||||
EXPECT_EQ(x, expected[i]);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(MindDataTestVocab, TestVocabFromMapFail) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestVocab-TestVocabFromMapFail.";
|
||||
// Build a map
|
||||
std::unordered_map<std::string, int32_t> dict;
|
||||
dict["banana"] = 0;
|
||||
dict["apple"] = -1;
|
||||
|
||||
// Expected failure: index of word can not be negative
|
||||
std::shared_ptr<Vocab> vocab = std::make_shared<Vocab>();
|
||||
Status s = Vocab::BuildFromUnorderedMap(dict, &vocab);
|
||||
EXPECT_NE(s, Status::OK());
|
||||
}
|
||||
|
||||
TEST_F(MindDataTestVocab, TestVocabFromVectorPrependSpTokens) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestVocab-TestVocabFromVectorPrependSpTokens.";
|
||||
// Build vocab from a vector of words, special tokens are prepended to vocab
|
||||
std::vector<std::string> list = {"apple", "banana", "cat", "dog", "egg"};
|
||||
std::shared_ptr<Vocab> vocab = std::make_shared<Vocab>();
|
||||
Status s = Vocab::BuildFromVector(list, {"<unk>"}, true, &vocab);
|
||||
EXPECT_EQ(s, Status::OK());
|
||||
|
||||
// Look up specified words
|
||||
// Expect that we will return -1 when word is not in vocab
|
||||
std::vector<std::string> words = {"apple", "banana", "fox"};
|
||||
std::vector<int32_t> expected = {1, 2, -1};
|
||||
for (uint32_t i = 0; i < words.size(); ++i) {
|
||||
int32_t x = vocab->Lookup(words[i]);
|
||||
EXPECT_EQ(x, expected[i]);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(MindDataTestVocab, TestVocabFromVectorAppendSpTokens) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestVocab-TestVocabFromVectorAppendSpTokens.";
|
||||
// Build vocab from a vector of words, special tokens are appended to vocab
|
||||
std::vector<std::string> list = {"apple", "banana", "cat", "dog", "egg"};
|
||||
std::shared_ptr<Vocab> vocab = std::make_shared<Vocab>();
|
||||
Status s = Vocab::BuildFromVector(list, {"<unk>"}, false, &vocab);
|
||||
EXPECT_EQ(s, Status::OK());
|
||||
|
||||
// Look up specified words
|
||||
std::vector<std::string> words = {"apple", "<unk>", "fox"};
|
||||
std::vector<int32_t> expected = {0, 5, -1};
|
||||
for (uint32_t i = 0; i < words.size(); ++i) {
|
||||
int32_t x = vocab->Lookup(words[i]);
|
||||
EXPECT_EQ(x, expected[i]);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(MindDataTestVocab, TestVocabFromVectorWithNoSpTokens) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestVocab-TestVocabFromVectorWithNoSpTokens.";
|
||||
// Build vocab from a vector of words with no special tokens
|
||||
std::vector<std::string> list = {"apple", "banana", "cat", "dog", "egg"};
|
||||
std::vector<std::string> sp_tokens = {};
|
||||
std::shared_ptr<Vocab> vocab = std::make_shared<Vocab>();
|
||||
Status s = Vocab::BuildFromVector(list, sp_tokens, true, &vocab);
|
||||
EXPECT_EQ(s, Status::OK());
|
||||
|
||||
// Look up specified words
|
||||
std::vector<std::string> words = {"apple", "banana", "fox", "<pad>"};
|
||||
std::vector<int32_t> expected = {0, 1, -1, -1};
|
||||
for (uint32_t i = 0; i < words.size(); ++i) {
|
||||
int32_t x = vocab->Lookup(words[i]);
|
||||
EXPECT_EQ(x, expected[i]);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(MindDataTestVocab, TestVocabFromEmptyVector) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestVocab-TestVocabFromEmptyVector.";
|
||||
// Build vocab from empty vector
|
||||
std::vector<std::string> list = {};
|
||||
std::shared_ptr<Vocab> vocab = std::make_shared<Vocab>();
|
||||
Status s = Vocab::BuildFromVector(list, {}, false, &vocab);
|
||||
EXPECT_EQ(s, Status::OK());
|
||||
|
||||
// Look up specified words
|
||||
// Expect that we will return -1 when word is not in vocab
|
||||
std::vector<std::string> words = {"apple", "banana", "fox"};
|
||||
std::vector<int32_t> expected = {-1, -1, -1};
|
||||
for (uint32_t i = 0; i < words.size(); ++i) {
|
||||
int32_t x = vocab->Lookup(words[i]);
|
||||
EXPECT_EQ(x, expected[i]);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(MindDataTestVocab, TestVocabFromVectorFail1) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestVocab-TestVocabFromVectorFail1.";
|
||||
// Build vocab from a vector of words with no special tokens
|
||||
std::vector<std::string> list = {"apple", "apple", "cat", "cat", "egg"};
|
||||
std::vector<std::string> sp_tokens = {};
|
||||
std::shared_ptr<Vocab> vocab = std::make_shared<Vocab>();
|
||||
|
||||
// Expected failure: duplicate word apple
|
||||
Status s = Vocab::BuildFromVector(list, sp_tokens, true, &vocab);
|
||||
EXPECT_NE(s, Status::OK());
|
||||
}
|
||||
|
||||
TEST_F(MindDataTestVocab, TestVocabFromVectorFail2) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestVocab-TestVocabFromVectorFail2.";
|
||||
// Build vocab from a vector of words with no special tokens
|
||||
std::vector<std::string> list = {"apple", "dog", "egg"};
|
||||
std::vector<std::string> sp_tokens = {"<pad>", "<unk>", "<pad>", "<unk>", "<none>"};
|
||||
std::shared_ptr<Vocab> vocab = std::make_shared<Vocab>();
|
||||
|
||||
// Expected failure: duplicate special token <pad> <unk>
|
||||
Status s = Vocab::BuildFromVector(list, sp_tokens, true, &vocab);
|
||||
EXPECT_NE(s, Status::OK());
|
||||
}
|
||||
|
||||
TEST_F(MindDataTestVocab, TestVocabFromFile) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestVocab-TestVocabFromFile.";
|
||||
// Build vocab from local file
|
||||
std::string vocab_dir = datasets_root_path_ + "/testVocab/vocab_list.txt";
|
||||
std::shared_ptr<Vocab> vocab = std::make_shared<Vocab>();
|
||||
Status s = Vocab::BuildFromFileCpp(vocab_dir, ",", -1, {"<pad>", "<unk>"}, true, &vocab);
|
||||
EXPECT_EQ(s, Status::OK());
|
||||
|
||||
// Look up specified words
|
||||
std::vector<std::string> words = {"not", "all"};
|
||||
std::vector<int32_t> expected = {2, 3};
|
||||
for (uint32_t i = 0; i < words.size(); ++i) {
|
||||
int32_t x = vocab->Lookup(words[i]);
|
||||
EXPECT_EQ(x, expected[i]);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(MindDataTestVocab, TestVocabFromFileFail1) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestVocab-TestVocabFromFileFail1.";
|
||||
// Build vocab from local file which is not exist
|
||||
std::string vocab_dir = datasets_root_path_ + "/testVocab/not_exist.txt";
|
||||
std::shared_ptr<Vocab> vocab = std::make_shared<Vocab>();
|
||||
Status s = Vocab::BuildFromFileCpp(vocab_dir, ",", -1, {}, true, &vocab);
|
||||
EXPECT_NE(s, Status::OK());
|
||||
}
|
||||
|
||||
TEST_F(MindDataTestVocab, TestVocabFromFileFail2) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestVocab-TestVocabFromFileFail2.";
|
||||
// Build vocab from local file
|
||||
std::string vocab_dir = datasets_root_path_ + "/testVocab/vocab_list.txt";
|
||||
std::shared_ptr<Vocab> vocab = std::make_shared<Vocab>();
|
||||
|
||||
// Expected failure: vocab_size shoule be either -1 or positive integer
|
||||
Status s = Vocab::BuildFromFileCpp(vocab_dir, ",", -2, {}, true, &vocab);
|
||||
EXPECT_NE(s, Status::OK());
|
||||
}
|
||||
|
||||
TEST_F(MindDataTestVocab, TestVocabFromFileFail3) {
|
||||
MS_LOG(INFO) << "Doing MindDataTestVocab-TestVocabFromFileFail2.";
|
||||
// Build vocab from local file which is not exist
|
||||
std::string vocab_dir = datasets_root_path_ + "/testVocab/vocab_list.txt";
|
||||
std::shared_ptr<Vocab> vocab = std::make_shared<Vocab>();
|
||||
|
||||
// Expected failure: duplicate special token <unk>
|
||||
Status s = Vocab::BuildFromFileCpp(vocab_dir, ",", -1, {"<unk>", "<unk>"}, true, &vocab);
|
||||
EXPECT_NE(s, Status::OK());
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
Loading…
Reference in new issue