remove old defaults

pull/4314/head
nhussain 5 years ago
parent ea60b3b52a
commit eb9a611041

@ -32,45 +32,37 @@ namespace mindspore {
namespace dataset { namespace dataset {
PYBIND_REGISTER(ConcatenateOp, 1, ([](const py::module *m) { PYBIND_REGISTER(ConcatenateOp, 1, ([](const py::module *m) {
(void)py::class_<ConcatenateOp, TensorOp, std::shared_ptr<ConcatenateOp>>( (void)py::class_<ConcatenateOp, TensorOp, std::shared_ptr<ConcatenateOp>>(*m, "ConcatenateOp")
*m, "ConcatenateOp", "Tensor operation concatenate tensors.") .def(py::init<int8_t, std::shared_ptr<Tensor>, std::shared_ptr<Tensor>>());
.def(py::init<int8_t, std::shared_ptr<Tensor>, std::shared_ptr<Tensor>>(), py::arg("axis"),
py::arg("prepend").none(true), py::arg("append").none(true));
})); }));
PYBIND_REGISTER(DuplicateOp, 1, ([](const py::module *m) { PYBIND_REGISTER(
(void)py::class_<DuplicateOp, TensorOp, std::shared_ptr<DuplicateOp>>(*m, "DuplicateOp", DuplicateOp, 1, ([](const py::module *m) {
"Duplicate tensor.") (void)py::class_<DuplicateOp, TensorOp, std::shared_ptr<DuplicateOp>>(*m, "DuplicateOp").def(py::init<>());
.def(py::init<>());
})); }));
PYBIND_REGISTER(FillOp, 1, ([](const py::module *m) { PYBIND_REGISTER(
(void)py::class_<FillOp, TensorOp, std::shared_ptr<FillOp>>( FillOp, 1, ([](const py::module *m) {
*m, "FillOp", "Tensor operation to return tensor filled with same value as input fill value.") (void)py::class_<FillOp, TensorOp, std::shared_ptr<FillOp>>(*m, "FillOp").def(py::init<std::shared_ptr<Tensor>>());
.def(py::init<std::shared_ptr<Tensor>>());
})); }));
PYBIND_REGISTER(MaskOp, 1, ([](const py::module *m) { PYBIND_REGISTER(MaskOp, 1, ([](const py::module *m) {
(void)py::class_<MaskOp, TensorOp, std::shared_ptr<MaskOp>>( (void)py::class_<MaskOp, TensorOp, std::shared_ptr<MaskOp>>(*m, "MaskOp")
*m, "MaskOp", "Tensor mask operation using relational comparator")
.def(py::init<RelationalOp, std::shared_ptr<Tensor>, DataType>()); .def(py::init<RelationalOp, std::shared_ptr<Tensor>, DataType>());
})); }));
PYBIND_REGISTER(OneHotOp, 1, ([](const py::module *m) { PYBIND_REGISTER(
(void)py::class_<OneHotOp, TensorOp, std::shared_ptr<OneHotOp>>( OneHotOp, 1, ([](const py::module *m) {
*m, "OneHotOp", "Tensor operation to apply one hot encoding. Takes number of classes.") (void)py::class_<OneHotOp, TensorOp, std::shared_ptr<OneHotOp>>(*m, "OneHotOp").def(py::init<int32_t>());
.def(py::init<int32_t>());
})); }));
PYBIND_REGISTER(PadEndOp, 1, ([](const py::module *m) { PYBIND_REGISTER(PadEndOp, 1, ([](const py::module *m) {
(void)py::class_<PadEndOp, TensorOp, std::shared_ptr<PadEndOp>>( (void)py::class_<PadEndOp, TensorOp, std::shared_ptr<PadEndOp>>(*m, "PadEndOp")
*m, "PadEndOp", "Tensor operation to pad end of tensor with a pad value.")
.def(py::init<TensorShape, std::shared_ptr<Tensor>>()); .def(py::init<TensorShape, std::shared_ptr<Tensor>>());
})); }));
PYBIND_REGISTER(SliceOp, 1, ([](const py::module *m) { PYBIND_REGISTER(SliceOp, 1, ([](const py::module *m) {
(void)py::class_<SliceOp, TensorOp, std::shared_ptr<SliceOp>>(*m, "SliceOp", (void)py::class_<SliceOp, TensorOp, std::shared_ptr<SliceOp>>(*m, "SliceOp")
"Tensor slice operation.")
.def(py::init<bool>()) .def(py::init<bool>())
.def(py::init([](const py::list &py_list) { .def(py::init([](const py::list &py_list) {
std::vector<dsize_t> c_list; std::vector<dsize_t> c_list;
@ -105,17 +97,15 @@ PYBIND_REGISTER(SliceOp, 1, ([](const py::module *m) {
})); }));
PYBIND_REGISTER(ToFloat16Op, 1, ([](const py::module *m) { PYBIND_REGISTER(ToFloat16Op, 1, ([](const py::module *m) {
(void)py::class_<ToFloat16Op, TensorOp, std::shared_ptr<ToFloat16Op>>( (void)py::class_<ToFloat16Op, TensorOp, std::shared_ptr<ToFloat16Op>>(*m, "ToFloat16Op",
*m, "ToFloat16Op", py::dynamic_attr(), py::dynamic_attr())
"Tensor operator to type cast float32 data to a float16 type.")
.def(py::init<>()); .def(py::init<>());
})); }));
PYBIND_REGISTER(TypeCastOp, 1, ([](const py::module *m) { PYBIND_REGISTER(TypeCastOp, 1, ([](const py::module *m) {
(void)py::class_<TypeCastOp, TensorOp, std::shared_ptr<TypeCastOp>>( (void)py::class_<TypeCastOp, TensorOp, std::shared_ptr<TypeCastOp>>(*m, "TypeCastOp")
*m, "TypeCastOp", "Tensor operator to type cast data to a specified type.") .def(py::init<DataType>())
.def(py::init<DataType>(), py::arg("data_type")) .def(py::init<std::string>());
.def(py::init<std::string>(), py::arg("data_type"));
})); }));
PYBIND_REGISTER(RelationalOp, 0, ([](const py::module *m) { PYBIND_REGISTER(RelationalOp, 0, ([](const py::module *m) {

@ -46,73 +46,50 @@ namespace dataset {
#ifdef ENABLE_ICU4C #ifdef ENABLE_ICU4C
PYBIND_REGISTER(BasicTokenizerOp, 1, ([](const py::module *m) { PYBIND_REGISTER(BasicTokenizerOp, 1, ([](const py::module *m) {
(void)py::class_<BasicTokenizerOp, TensorOp, std::shared_ptr<BasicTokenizerOp>>( (void)py::class_<BasicTokenizerOp, TensorOp, std::shared_ptr<BasicTokenizerOp>>(*m,
*m, "BasicTokenizerOp", "Tokenize a scalar tensor of UTF-8 string by specific rules.") "BasicTokenizerOp")
.def(py::init<const bool &, const bool &, const NormalizeForm &, const bool &, const bool &>(), .def(py::init<const bool &, const bool &, const NormalizeForm &, const bool &, const bool &>());
py::arg("lower_case") = BasicTokenizerOp::kDefLowerCase,
py::arg("keep_whitespace") = BasicTokenizerOp::kDefKeepWhitespace,
py::arg("normalization_form") = BasicTokenizerOp::kDefNormalizationForm,
py::arg("preserve_unused_token") = BasicTokenizerOp::kDefPreserveUnusedToken,
py::arg("with_offsets") = BasicTokenizerOp::kDefWithOffsets);
})); }));
PYBIND_REGISTER(WhitespaceTokenizerOp, 1, ([](const py::module *m) { PYBIND_REGISTER(WhitespaceTokenizerOp, 1, ([](const py::module *m) {
(void)py::class_<WhitespaceTokenizerOp, TensorOp, std::shared_ptr<WhitespaceTokenizerOp>>( (void)py::class_<WhitespaceTokenizerOp, TensorOp, std::shared_ptr<WhitespaceTokenizerOp>>(
*m, "WhitespaceTokenizerOp", "Tokenize a scalar tensor of UTF-8 string on ICU defined whitespaces.") *m, "WhitespaceTokenizerOp")
.def(py::init<const bool &>(), py::arg(" with_offsets ") = WhitespaceTokenizerOp::kDefWithOffsets); .def(py::init<const bool &>());
})); }));
PYBIND_REGISTER(UnicodeScriptTokenizerOp, 1, ([](const py::module *m) { PYBIND_REGISTER(UnicodeScriptTokenizerOp, 1, ([](const py::module *m) {
(void)py::class_<UnicodeScriptTokenizerOp, TensorOp, std::shared_ptr<UnicodeScriptTokenizerOp>>( (void)py::class_<UnicodeScriptTokenizerOp, TensorOp, std::shared_ptr<UnicodeScriptTokenizerOp>>(
*m, "UnicodeScriptTokenizerOp", *m, "UnicodeScriptTokenizerOp")
"Tokenize a scalar tensor of UTF-8 string on Unicode script boundaries.")
.def(py::init<>()) .def(py::init<>())
.def(py::init<const bool &, const bool &>(), .def(py::init<const bool &, const bool &>());
py::arg("keep_whitespace") = UnicodeScriptTokenizerOp::kDefKeepWhitespace,
py::arg("with_offsets") = UnicodeScriptTokenizerOp::kDefWithOffsets);
})); }));
PYBIND_REGISTER(CaseFoldOp, 1, ([](const py::module *m) { PYBIND_REGISTER(
(void)py::class_<CaseFoldOp, TensorOp, std::shared_ptr<CaseFoldOp>>( CaseFoldOp, 1, ([](const py::module *m) {
*m, "CaseFoldOp", "Apply case fold operation on utf-8 string tensor") (void)py::class_<CaseFoldOp, TensorOp, std::shared_ptr<CaseFoldOp>>(*m, "CaseFoldOp").def(py::init<>());
.def(py::init<>());
})); }));
PYBIND_REGISTER(NormalizeUTF8Op, 1, ([](const py::module *m) { PYBIND_REGISTER(NormalizeUTF8Op, 1, ([](const py::module *m) {
(void)py::class_<NormalizeUTF8Op, TensorOp, std::shared_ptr<NormalizeUTF8Op>>( (void)py::class_<NormalizeUTF8Op, TensorOp, std::shared_ptr<NormalizeUTF8Op>>(*m, "NormalizeUTF8Op")
*m, "NormalizeUTF8Op", "Apply normalize operation on utf-8 string tensor.")
.def(py::init<>()) .def(py::init<>())
.def(py::init<NormalizeForm>(), py::arg("normalize_form") = NormalizeUTF8Op::kDefNormalizeForm); .def(py::init<NormalizeForm>());
})); }));
PYBIND_REGISTER(RegexReplaceOp, 1, ([](const py::module *m) { PYBIND_REGISTER(RegexReplaceOp, 1, ([](const py::module *m) {
(void)py::class_<RegexReplaceOp, TensorOp, std::shared_ptr<RegexReplaceOp>>( (void)py::class_<RegexReplaceOp, TensorOp, std::shared_ptr<RegexReplaceOp>>(*m, "RegexReplaceOp")
*m, "RegexReplaceOp", .def(py::init<const std::string &, const std::string &, bool>());
"Replace utf-8 string tensor with 'replace' according to regular expression 'pattern'.")
.def(py::init<const std::string &, const std::string &, bool>(), py::arg("pattern"),
py::arg("replace"), py::arg("replace_all"));
})); }));
PYBIND_REGISTER(RegexTokenizerOp, 1, ([](const py::module *m) { PYBIND_REGISTER(RegexTokenizerOp, 1, ([](const py::module *m) {
(void)py::class_<RegexTokenizerOp, TensorOp, std::shared_ptr<RegexTokenizerOp>>( (void)py::class_<RegexTokenizerOp, TensorOp, std::shared_ptr<RegexTokenizerOp>>(*m,
*m, "RegexTokenizerOp", "Tokenize a scalar tensor of UTF-8 string by regex expression pattern.") "RegexTokenizerOp")
.def(py::init<const std::string &, const std::string &, const bool &>(), py::arg("delim_pattern"), .def(py::init<const std::string &, const std::string &, const bool &>());
py::arg("keep_delim_pattern"), py::arg("with_offsets") = RegexTokenizerOp::kDefWithOffsets);
})); }));
PYBIND_REGISTER(BertTokenizerOp, 1, ([](const py::module *m) { PYBIND_REGISTER(BertTokenizerOp, 1, ([](const py::module *m) {
(void)py::class_<BertTokenizerOp, TensorOp, std::shared_ptr<BertTokenizerOp>>( (void)py::class_<BertTokenizerOp, TensorOp, std::shared_ptr<BertTokenizerOp>>(*m, "BertTokenizerOp")
*m, "BertTokenizerOp", "Tokenizer used for Bert text process.")
.def(py::init<const std::shared_ptr<Vocab> &, const std::string &, const int &, const std::string &, .def(py::init<const std::shared_ptr<Vocab> &, const std::string &, const int &, const std::string &,
const bool &, const bool &, const NormalizeForm &, const bool &, const bool &>(), const bool &, const bool &, const NormalizeForm &, const bool &, const bool &>());
py::arg("vocab"),
py::arg("suffix_indicator") = std::string(WordpieceTokenizerOp::kDefSuffixIndicator),
py::arg("max_bytes_per_token") = WordpieceTokenizerOp::kDefMaxBytesPerToken,
py::arg("unknown_token") = std::string(WordpieceTokenizerOp::kDefUnknownToken),
py::arg("lower_case") = BasicTokenizerOp::kDefLowerCase,
py::arg("keep_whitespace") = BasicTokenizerOp::kDefKeepWhitespace,
py::arg("normalization_form") = BasicTokenizerOp::kDefNormalizationForm,
py::arg("preserve_unused_token") = BasicTokenizerOp::kDefPreserveUnusedToken,
py::arg("with_offsets") = WordpieceTokenizerOp::kDefWithOffsets);
})); }));
PYBIND_REGISTER(NormalizeForm, 0, ([](const py::module *m) { PYBIND_REGISTER(NormalizeForm, 0, ([](const py::module *m) {
@ -128,11 +105,9 @@ PYBIND_REGISTER(NormalizeForm, 0, ([](const py::module *m) {
#endif #endif
PYBIND_REGISTER(JiebaTokenizerOp, 1, ([](const py::module *m) { PYBIND_REGISTER(JiebaTokenizerOp, 1, ([](const py::module *m) {
(void)py::class_<JiebaTokenizerOp, TensorOp, std::shared_ptr<JiebaTokenizerOp>>( (void)py::class_<JiebaTokenizerOp, TensorOp, std::shared_ptr<JiebaTokenizerOp>>(*m,
*m, "JiebaTokenizerOp", "") "JiebaTokenizerOp")
.def(py::init<const std::string &, const std::string &, const JiebaMode &, const bool &>(), .def(py::init<const std::string &, const std::string &, const JiebaMode &, const bool &>())
py::arg("hmm_path"), py::arg("mp_path"), py::arg("mode") = JiebaMode::kMix,
py::arg("with_offsets") = JiebaTokenizerOp::kDefWithOffsets)
.def("add_word", [](JiebaTokenizerOp &self, const std::string word, int freq) { .def("add_word", [](JiebaTokenizerOp &self, const std::string word, int freq) {
THROW_IF_ERROR(self.AddWord(word, freq)); THROW_IF_ERROR(self.AddWord(word, freq));
}); });
@ -140,13 +115,12 @@ PYBIND_REGISTER(JiebaTokenizerOp, 1, ([](const py::module *m) {
PYBIND_REGISTER(UnicodeCharTokenizerOp, 1, ([](const py::module *m) { PYBIND_REGISTER(UnicodeCharTokenizerOp, 1, ([](const py::module *m) {
(void)py::class_<UnicodeCharTokenizerOp, TensorOp, std::shared_ptr<UnicodeCharTokenizerOp>>( (void)py::class_<UnicodeCharTokenizerOp, TensorOp, std::shared_ptr<UnicodeCharTokenizerOp>>(
*m, "UnicodeCharTokenizerOp", "Tokenize a scalar tensor of UTF-8 string to Unicode characters.") *m, "UnicodeCharTokenizerOp")
.def(py::init<const bool &>(), py::arg("with_offsets") = UnicodeCharTokenizerOp::kDefWithOffsets); .def(py::init<const bool &>());
})); }));
PYBIND_REGISTER(LookupOp, 1, ([](const py::module *m) { PYBIND_REGISTER(LookupOp, 1, ([](const py::module *m) {
(void)py::class_<LookupOp, TensorOp, std::shared_ptr<LookupOp>>( (void)py::class_<LookupOp, TensorOp, std::shared_ptr<LookupOp>>(*m, "LookupOp")
*m, "LookupOp", "Tensor operation to LookUp each word.")
.def(py::init([](std::shared_ptr<Vocab> vocab, const py::object &py_word) { .def(py::init([](std::shared_ptr<Vocab> vocab, const py::object &py_word) {
if (vocab == nullptr) { if (vocab == nullptr) {
THROW_IF_ERROR(Status(StatusCode::kUnexpectedError, "vocab object type is incorrect or null.")); THROW_IF_ERROR(Status(StatusCode::kUnexpectedError, "vocab object type is incorrect or null."));
@ -165,56 +139,42 @@ PYBIND_REGISTER(LookupOp, 1, ([](const py::module *m) {
})); }));
PYBIND_REGISTER(NgramOp, 1, ([](const py::module *m) { PYBIND_REGISTER(NgramOp, 1, ([](const py::module *m) {
(void)py::class_<NgramOp, TensorOp, std::shared_ptr<NgramOp>>(*m, "NgramOp", (void)py::class_<NgramOp, TensorOp, std::shared_ptr<NgramOp>>(*m, "NgramOp")
"TensorOp performs ngram mapping.")
.def(py::init<const std::vector<int32_t> &, int32_t, int32_t, const std::string &, .def(py::init<const std::vector<int32_t> &, int32_t, int32_t, const std::string &,
const std::string &, const std::string &>(), const std::string &, const std::string &>());
py::arg("ngrams"), py::arg("l_pad_len"), py::arg("r_pad_len"), py::arg("l_pad_token"),
py::arg("r_pad_token"), py::arg("separator"));
})); }));
PYBIND_REGISTER( PYBIND_REGISTER(WordpieceTokenizerOp, 1, ([](const py::module *m) {
WordpieceTokenizerOp, 1, ([](const py::module *m) {
(void)py::class_<WordpieceTokenizerOp, TensorOp, std::shared_ptr<WordpieceTokenizerOp>>( (void)py::class_<WordpieceTokenizerOp, TensorOp, std::shared_ptr<WordpieceTokenizerOp>>(
*m, "WordpieceTokenizerOp", "Tokenize scalar token or 1-D tokens to subword tokens.") *m, "WordpieceTokenizerOp")
.def( .def(py::init<const std::shared_ptr<Vocab> &, const std::string &, const int &, const std::string &,
py::init<const std::shared_ptr<Vocab> &, const std::string &, const int &, const std::string &, const bool &>(), const bool &>());
py::arg("vocab"), py::arg("suffix_indicator") = std::string(WordpieceTokenizerOp::kDefSuffixIndicator),
py::arg("max_bytes_per_token") = WordpieceTokenizerOp::kDefMaxBytesPerToken,
py::arg("unknown_token") = std::string(WordpieceTokenizerOp::kDefUnknownToken),
py::arg("with_offsets") = WordpieceTokenizerOp::kDefWithOffsets);
})); }));
PYBIND_REGISTER(SlidingWindowOp, 1, ([](const py::module *m) { PYBIND_REGISTER(SlidingWindowOp, 1, ([](const py::module *m) {
(void)py::class_<SlidingWindowOp, TensorOp, std::shared_ptr<SlidingWindowOp>>( (void)py::class_<SlidingWindowOp, TensorOp, std::shared_ptr<SlidingWindowOp>>(*m, "SlidingWindowOp")
*m, "SlidingWindowOp", "TensorOp to apply sliding window to a 1-D Tensor.") .def(py::init<uint32_t, int32_t>());
.def(py::init<uint32_t, int32_t>(), py::arg("width"), py::arg("axis"));
})); }));
PYBIND_REGISTER( PYBIND_REGISTER(
SentencePieceTokenizerOp, 1, ([](const py::module *m) { SentencePieceTokenizerOp, 1, ([](const py::module *m) {
(void)py::class_<SentencePieceTokenizerOp, TensorOp, std::shared_ptr<SentencePieceTokenizerOp>>( (void)py::class_<SentencePieceTokenizerOp, TensorOp, std::shared_ptr<SentencePieceTokenizerOp>>(
*m, "SentencePieceTokenizerOp", "Tokenize scalar token or 1-D tokens to tokens by sentence piece.") *m, "SentencePieceTokenizerOp")
.def( .def(
py::init<std::shared_ptr<SentencePieceVocab> &, const SPieceTokenizerLoadType, const SPieceTokenizerOutType>(), py::init<std::shared_ptr<SentencePieceVocab> &, const SPieceTokenizerLoadType, const SPieceTokenizerOutType>())
py::arg("vocab"), py::arg("load_type") = SPieceTokenizerLoadType::kModel,
py::arg("out_type") = SPieceTokenizerOutType::kString)
.def(py::init<const std::string &, const std::string &, const SPieceTokenizerLoadType, .def(py::init<const std::string &, const std::string &, const SPieceTokenizerLoadType,
const SPieceTokenizerOutType>(), const SPieceTokenizerOutType>());
py::arg("model_path"), py::arg("model_filename"), py::arg("load_type") = SPieceTokenizerLoadType::kFile,
py::arg("out_type") = SPieceTokenizerOutType::kString);
})); }));
PYBIND_REGISTER(ToNumberOp, 1, ([](const py::module *m) { PYBIND_REGISTER(ToNumberOp, 1, ([](const py::module *m) {
(void)py::class_<ToNumberOp, TensorOp, std::shared_ptr<ToNumberOp>>( (void)py::class_<ToNumberOp, TensorOp, std::shared_ptr<ToNumberOp>>(*m, "ToNumberOp")
*m, "ToNumberOp", "TensorOp to convert strings to numbers.") .def(py::init<DataType>())
.def(py::init<DataType>(), py::arg("data_type")) .def(py::init<std::string>());
.def(py::init<std::string>(), py::arg("data_type"));
})); }));
PYBIND_REGISTER(TruncateSequencePairOp, 1, ([](const py::module *m) { PYBIND_REGISTER(TruncateSequencePairOp, 1, ([](const py::module *m) {
(void)py::class_<TruncateSequencePairOp, TensorOp, std::shared_ptr<TruncateSequencePairOp>>( (void)py::class_<TruncateSequencePairOp, TensorOp, std::shared_ptr<TruncateSequencePairOp>>(
*m, "TruncateSequencePairOp", "Tensor operation to truncate two tensors to a max_length") *m, "TruncateSequencePairOp")
.def(py::init<int64_t>()); .def(py::init<int64_t>());
})); }));

@ -62,7 +62,7 @@ PYBIND_REGISTER(
ShardSample, 0, ([](const py::module *m) { ShardSample, 0, ([](const py::module *m) {
(void)py::class_<mindrecord::ShardSample, mindrecord::ShardOperator, std::shared_ptr<mindrecord::ShardSample>>( (void)py::class_<mindrecord::ShardSample, mindrecord::ShardOperator, std::shared_ptr<mindrecord::ShardSample>>(
*m, "MindrecordSubsetRandomSampler") *m, "MindrecordSubsetRandomSampler")
.def(py::init<std::vector<int64_t>, uint32_t>(), py::arg("indices"), py::arg("seed") = GetSeed()); .def(py::init<std::vector<int64_t>, uint32_t>());
})); }));
PYBIND_REGISTER(ShardSequentialSample, 0, ([](const py::module *m) { PYBIND_REGISTER(ShardSequentialSample, 0, ([](const py::module *m) {

@ -37,7 +37,6 @@ add_library(kernels-image OBJECT
random_vertical_flip_with_bbox_op.cc random_vertical_flip_with_bbox_op.cc
random_sharpness_op.cc random_sharpness_op.cc
rescale_op.cc rescale_op.cc
resize_bilinear_op.cc
resize_op.cc resize_op.cc
rgba_to_bgr_op.cc rgba_to_bgr_op.cc
rgba_to_rgb_op.cc rgba_to_rgb_op.cc

@ -1,26 +0,0 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "minddata/dataset/kernels/image/resize_bilinear_op.h"
#include <random>
#include "minddata/dataset/util/status.h"
namespace mindspore {
namespace dataset {
const int32_t ResizeBilinearOp::kDefWidth = 0;
} // namespace dataset
} // namespace mindspore

@ -21,6 +21,7 @@ User can also define custom sampler by extending from Sampler class.
import numpy as np import numpy as np
import mindspore._c_dataengine as cde import mindspore._c_dataengine as cde
import mindspore.dataset as ds
class Sampler: class Sampler:
""" """
@ -541,7 +542,7 @@ class SubsetRandomSampler(BuiltinSampler):
return self.child_sampler.is_sharded() return self.child_sampler.is_sharded()
def create_for_minddataset(self): def create_for_minddataset(self):
c_sampler = cde.MindrecordSubsetRandomSampler(self.indices) c_sampler = cde.MindrecordSubsetRandomSampler(self.indices, ds.config.get_seed())
c_child_sampler = self.create_child_for_minddataset() c_child_sampler = self.create_child_for_minddataset()
c_sampler.add_child(c_child_sampler) c_sampler.add_child(c_child_sampler)
return c_sampler return c_sampler

@ -52,8 +52,8 @@ import mindspore._c_dataengine as cde
from .utils import JiebaMode, NormalizeForm, to_str, SPieceTokenizerOutType, SPieceTokenizerLoadType from .utils import JiebaMode, NormalizeForm, to_str, SPieceTokenizerOutType, SPieceTokenizerLoadType
from .validators import check_lookup, check_jieba_add_dict, \ from .validators import check_lookup, check_jieba_add_dict, \
check_jieba_add_word, check_jieba_init, check_with_offsets, check_unicode_script_tokenizer,\ check_jieba_add_word, check_jieba_init, check_with_offsets, check_unicode_script_tokenizer, \
check_wordpiece_tokenizer, check_regex_tokenizer, check_basic_tokenizer, check_ngram, check_pair_truncate,\ check_wordpiece_tokenizer, check_regex_tokenizer, check_basic_tokenizer, check_ngram, check_pair_truncate, \
check_to_number, check_bert_tokenizer, check_python_tokenizer, check_slidingwindow check_to_number, check_bert_tokenizer, check_python_tokenizer, check_slidingwindow
from ..core.datatypes import mstype_to_detype from ..core.datatypes import mstype_to_detype
@ -100,7 +100,8 @@ class SlidingWindow(cde.SlidingWindowOp):
@check_slidingwindow @check_slidingwindow
def __init__(self, width, axis=0): def __init__(self, width, axis=0):
super().__init__(width=width, axis=axis) super().__init__(width, axis)
class Ngram(cde.NgramOp): class Ngram(cde.NgramOp):
@ -126,8 +127,7 @@ class Ngram(cde.NgramOp):
@check_ngram @check_ngram
def __init__(self, n, left_pad=("", 0), right_pad=("", 0), separator=" "): def __init__(self, n, left_pad=("", 0), right_pad=("", 0), separator=" "):
super().__init__(ngrams=n, l_pad_len=left_pad[1], r_pad_len=right_pad[1], l_pad_token=left_pad[0], super().__init__(n, left_pad[1], right_pad[1], left_pad[0], right_pad[0], separator)
r_pad_token=right_pad[0], separator=separator)
DE_C_INTER_JIEBA_MODE = { DE_C_INTER_JIEBA_MODE = {
@ -326,6 +326,7 @@ class WordpieceTokenizer(cde.WordpieceTokenizerOp):
super().__init__(self.vocab, self.suffix_indicator, self.max_bytes_per_token, super().__init__(self.vocab, self.suffix_indicator, self.max_bytes_per_token,
self.unknown_token, self.with_offsets) self.unknown_token, self.with_offsets)
DE_C_INTER_SENTENCEPIECE_LOADTYPE = { DE_C_INTER_SENTENCEPIECE_LOADTYPE = {
SPieceTokenizerLoadType.FILE: cde.SPieceTokenizerLoadType.DE_SPIECE_TOKENIZER_LOAD_KFILE, SPieceTokenizerLoadType.FILE: cde.SPieceTokenizerLoadType.DE_SPIECE_TOKENIZER_LOAD_KFILE,
SPieceTokenizerLoadType.MODEL: cde.SPieceTokenizerLoadType.DE_SPIECE_TOKENIZER_LOAD_KMODEL SPieceTokenizerLoadType.MODEL: cde.SPieceTokenizerLoadType.DE_SPIECE_TOKENIZER_LOAD_KMODEL
@ -336,6 +337,7 @@ DE_C_INTER_SENTENCEPIECE_OUTTYPE = {
SPieceTokenizerOutType.INT: cde.SPieceTokenizerOutType.DE_SPIECE_TOKENIZER_OUTTYPE_KINT SPieceTokenizerOutType.INT: cde.SPieceTokenizerOutType.DE_SPIECE_TOKENIZER_OUTTYPE_KINT
} }
class SentencePieceTokenizer(cde.SentencePieceTokenizerOp): class SentencePieceTokenizer(cde.SentencePieceTokenizerOp):
""" """
Tokenize scalar token or 1-D tokens to tokens by sentencepiece. Tokenize scalar token or 1-D tokens to tokens by sentencepiece.
@ -357,6 +359,7 @@ class SentencePieceTokenizer(cde.SentencePieceTokenizerOp):
super().__init__(mode, DE_C_INTER_SENTENCEPIECE_LOADTYPE[SPieceTokenizerLoadType.MODEL], super().__init__(mode, DE_C_INTER_SENTENCEPIECE_LOADTYPE[SPieceTokenizerLoadType.MODEL],
DE_C_INTER_SENTENCEPIECE_OUTTYPE[out_type]) DE_C_INTER_SENTENCEPIECE_OUTTYPE[out_type])
if platform.system().lower() != 'windows': if platform.system().lower() != 'windows':
class WhitespaceTokenizer(cde.WhitespaceTokenizerOp): class WhitespaceTokenizer(cde.WhitespaceTokenizerOp):
""" """

@ -362,7 +362,7 @@ class RandomCrop(cde.RandomCropOp):
padding = (0, 0, 0, 0) padding = (0, 0, 0, 0)
else: else:
padding = parse_padding(padding) padding = parse_padding(padding)
if isinstance(fill_value, int): # temporary fix if isinstance(fill_value, int):
fill_value = tuple([fill_value] * 3) fill_value = tuple([fill_value] * 3)
border_type = DE_C_BORDER_TYPE[padding_mode] border_type = DE_C_BORDER_TYPE[padding_mode]
@ -417,7 +417,7 @@ class RandomCropWithBBox(cde.RandomCropWithBBoxOp):
else: else:
padding = parse_padding(padding) padding = parse_padding(padding)
if isinstance(fill_value, int): # temporary fix if isinstance(fill_value, int):
fill_value = tuple([fill_value] * 3) fill_value = tuple([fill_value] * 3)
border_type = DE_C_BORDER_TYPE[padding_mode] border_type = DE_C_BORDER_TYPE[padding_mode]
@ -549,8 +549,7 @@ class Resize(cde.ResizeOp):
self.interpolation = interpolation self.interpolation = interpolation
interpoltn = DE_C_INTER_MODE[interpolation] interpoltn = DE_C_INTER_MODE[interpolation]
if isinstance(size, int): if isinstance(size, int):
super().__init__(size, interpolation=interpoltn) size = (size, 0)
else:
super().__init__(*size, interpoltn) super().__init__(*size, interpoltn)
@ -579,8 +578,7 @@ class ResizeWithBBox(cde.ResizeWithBBoxOp):
self.interpolation = interpolation self.interpolation = interpolation
interpoltn = DE_C_INTER_MODE[interpolation] interpoltn = DE_C_INTER_MODE[interpolation]
if isinstance(size, int): if isinstance(size, int):
super().__init__(size, interpolation=interpoltn) size = (size, 0)
else:
super().__init__(*size, interpoltn) super().__init__(*size, interpoltn)
@ -779,7 +777,7 @@ class RandomRotation(cde.RandomRotationOp):
degrees = (-degrees, degrees) degrees = (-degrees, degrees)
if center is None: if center is None:
center = (-1, -1) center = (-1, -1)
if isinstance(fill_value, int): # temporary fix if isinstance(fill_value, int):
fill_value = tuple([fill_value] * 3) fill_value = tuple([fill_value] * 3)
interpolation = DE_C_INTER_MODE[resample] interpolation = DE_C_INTER_MODE[resample]
super().__init__(*degrees, *center, interpolation, expand, *fill_value) super().__init__(*degrees, *center, interpolation, expand, *fill_value)
@ -816,8 +814,7 @@ class RandomResize(cde.RandomResizeOp):
def __init__(self, size): def __init__(self, size):
self.size = size self.size = size
if isinstance(size, int): if isinstance(size, int):
super().__init__(size) size = (size, 0)
else:
super().__init__(*size) super().__init__(*size)
@ -837,8 +834,7 @@ class RandomResizeWithBBox(cde.RandomResizeWithBBoxOp):
def __init__(self, size): def __init__(self, size):
self.size = size self.size = size
if isinstance(size, int): if isinstance(size, int):
super().__init__(size) size = (size, 0)
else:
super().__init__(*size) super().__init__(*size)
@ -918,7 +914,7 @@ class Pad(cde.PadOp):
@check_pad @check_pad
def __init__(self, padding, fill_value=0, padding_mode=Border.CONSTANT): def __init__(self, padding, fill_value=0, padding_mode=Border.CONSTANT):
padding = parse_padding(padding) padding = parse_padding(padding)
if isinstance(fill_value, int): # temporary fix if isinstance(fill_value, int):
fill_value = tuple([fill_value] * 3) fill_value = tuple([fill_value] * 3)
padding_mode = DE_C_BORDER_TYPE[padding_mode] padding_mode = DE_C_BORDER_TYPE[padding_mode]

@ -58,7 +58,6 @@ SET(DE_UT_SRCS
rename_op_test.cc rename_op_test.cc
repeat_op_test.cc repeat_op_test.cc
rescale_op_test.cc rescale_op_test.cc
resize_bilinear_op_test.cc
resize_op_test.cc resize_op_test.cc
resize_with_bbox_op_test.cc resize_with_bbox_op_test.cc
rgba_to_bgr_op_test.cc rgba_to_bgr_op_test.cc

@ -1200,4 +1200,8 @@ TEST_F(MindDataTestPipeline, TestRandomSolarizeFail) {
threshold = {1}; threshold = {1};
random_solarize = mindspore::dataset::api::vision::RandomSolarize(threshold); random_solarize = mindspore::dataset::api::vision::RandomSolarize(threshold);
EXPECT_EQ(random_solarize, nullptr); EXPECT_EQ(random_solarize, nullptr);
threshold = {};
random_solarize = mindspore::dataset::api::vision::RandomSolarize(threshold);
EXPECT_EQ(random_solarize, nullptr);
} }

@ -1,44 +0,0 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "common/common.h"
#include "common/cvop_common.h"
#include "minddata/dataset/kernels/image/resize_bilinear_op.h"
#include "utils/log_adapter.h"
using namespace mindspore::dataset;
using mindspore::MsLogLevel::INFO;
using mindspore::ExceptionType::NoExceptionType;
using mindspore::LogStream;
class MindDataTestResizeBilinearOp : public UT::CVOP::CVOpCommon {
public:
MindDataTestResizeBilinearOp() : CVOpCommon() {}
};
TEST_F(MindDataTestResizeBilinearOp, TestOp) {
MS_LOG(INFO) << "Doing testResizeBilinear.";
// Resizing with a factor of 0.5
TensorShape s = input_tensor_->shape();
int output_w = 0.5 * s[0];
int output_h = (s[0] * output_w) / s[1];
std::shared_ptr<Tensor> output_tensor;
// Resizing
std::unique_ptr<ResizeBilinearOp> op(new ResizeBilinearOp(output_h, output_w));
Status st = op->Compute(input_tensor_, &output_tensor);
EXPECT_TRUE(st.IsOk());
CheckImageShapeAndData(output_tensor, kResizeBilinear);
MS_LOG(INFO) << "testResizeBilinear end.";
}
Loading…
Cancel
Save