add default for sentencepiece col_names processing and vocab_size must be provided

pull/7902/head
xulei2020 5 years ago
parent 7633727fc8
commit 90c1314ff1

@ -1959,6 +1959,8 @@ Status DEPipeline::ParseBuildSentencePieceVocabOp(const py::dict &args, std::sha
if (!value.is_none()) {
if (key == "vocab_size") {
builder->SetVocabSize(ToInt(value));
} else if (key == "columns") {
(void)builder->SetColumnNames(ToStringVector(value));
} else if (key == "character_coverage") {
(void)builder->SetCharacterCoverage(ToFloat(value));
} else if (key == "params") {

@ -436,6 +436,9 @@ def check_from_dataset_sentencepiece(method):
if vocab_size is not None:
check_uint32(vocab_size, "vocab_size")
else:
raise TypeError("vocab_size must be provided")
if character_coverage is not None:
type_check(character_coverage, (float,), "character_coverage")

Loading…
Cancel
Save