!7902 [MD]add default for sentencepiece col_names processing and vocab_size must be provided

Merge pull request !7902 from xulei/lite_test0905
pull/7902/MERGE
mindspore-ci-bot 5 years ago committed by Gitee
commit 066ff87ef4

@ -1959,6 +1959,8 @@ Status DEPipeline::ParseBuildSentencePieceVocabOp(const py::dict &args, std::sha
if (!value.is_none()) {
if (key == "vocab_size") {
builder->SetVocabSize(ToInt(value));
} else if (key == "columns") {
(void)builder->SetColumnNames(ToStringVector(value));
} else if (key == "character_coverage") {
(void)builder->SetCharacterCoverage(ToFloat(value));
} else if (key == "params") {

@ -436,6 +436,9 @@ def check_from_dataset_sentencepiece(method):
if vocab_size is not None:
check_uint32(vocab_size, "vocab_size")
else:
raise TypeError("vocab_size must be provided")
if character_coverage is not None:
type_check(character_coverage, (float,), "character_coverage")

Loading…
Cancel
Save