m53297601
/
mindspore
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

!9534 Remove std::optional in pybind

Browse Source 
From: @alex-yuyue
Reviewed-by: 
Signed-off-by:
pull/9534/MERGE
mindspore-ci-bot 4 years ago committed by Gitee
parent 49fd5308a4 797b37c78d
commit 66de0eec3a
4 changed files with 124 additions and 137 deletions
Show Stats Download Patch File Download Diff File

194
mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/include/datasets_bindings.cc
View File

File diff suppressed because it is too large Load Diff

40
mindspore/ccsrc/minddata/dataset/api/python/pybind_conversion.cc
Unescape View File

@ -28,10 +28,10 @@ bool toBool(const py::handle &handle) { return py::reinterpret_borrow<py::bool_>
std::string toString(const py::handle &handle) { return py::reinterpret_borrow<py::str>(handle); }
std::set<std::string> toStringSet(const std::optional<py::list> list) {
std::set<std::string> toStringSet(const py::list list) {
std::set<std::string> set;
if (list) {
for (auto l : *list) {
if (!list.empty()) {
for (auto l : list) {
if (!l.is_none()) {
(void)set.insert(py::str(l));
}
@ -40,20 +40,20 @@ std::set<std::string> toStringSet(const std::optional<py::list> list) {
return set;
}
std::map<std::string, int32_t> toStringMap(const std::optional<py::dict> dict) {
std::map<std::string, int32_t> toStringMap(const py::dict dict) {
std::map<std::string, int32_t> map;
if (dict) {
for (auto p : *dict) {
if (!dict.empty()) {
for (auto p : dict) {
(void)map.emplace(toString(p.first), toInt(p.second));
}
}
return map;
}
std::vector<std::string> toStringVector(const std::optional<py::list> list) {
std::vector<std::string> toStringVector(const py::list list) {
std::vector<std::string> vector;
if (list) {
for (auto l : *list) {
if (!list.empty()) {
for (auto l : list) {
if (l.is_none())
vector.emplace_back("");
else
@ -63,10 +63,10 @@ std::vector<std::string> toStringVector(const std::optional<py::list> list) {
return vector;
}
std::pair<int64_t, int64_t> toIntPair(const std::optional<py::tuple> tuple) {
std::pair<int64_t, int64_t> toIntPair(const py::tuple tuple) {
std::pair<int64_t, int64_t> pair;
if (tuple) {
pair = std::make_pair(toInt64((*tuple)[0]), toInt64((*tuple)[1]));
if (!tuple.empty()) {
pair = std::make_pair(toInt64((tuple)[0]), toInt64((tuple)[1]));
}
return pair;
}
@ -85,10 +85,10 @@ std::vector<std::pair<int, int>> toPairVector(const py::list list) {
return vector;
}
std::vector<std::shared_ptr<TensorOperation>> toTensorOperations(std::optional<py::list> operations) {
std::vector<std::shared_ptr<TensorOperation>> toTensorOperations(py::list operations) {
std::vector<std::shared_ptr<TensorOperation>> vector;
if (operations) {
for (auto op : *operations) {
if (!operations.empty()) {
for (auto op : operations) {
std::shared_ptr<TensorOp> tensor_op;
if (py::isinstance<TensorOp>(op)) {
tensor_op = op.cast<std::shared_ptr<TensorOp>>();
@ -132,19 +132,19 @@ std::vector<std::shared_ptr<DatasetNode>> toDatasetNode(std::shared_ptr<DatasetN
return vector;
}
std::shared_ptr<SamplerObj> toSamplerObj(std::optional<py::handle> py_sampler, bool isMindDataset) {
std::shared_ptr<SamplerObj> toSamplerObj(py::handle py_sampler, bool isMindDataset) {
if (py_sampler) {
std::shared_ptr<SamplerObj> sampler_obj;
if (!isMindDataset) {
// Common Sampler
std::shared_ptr<SamplerRT> sampler;
auto create = py::reinterpret_borrow<py::object>(py_sampler.value()).attr("create");
auto create = py::reinterpret_borrow<py::object>(py_sampler).attr("create");
sampler = create().cast<std::shared_ptr<SamplerRT>>();
sampler_obj = std::make_shared<PreBuiltSamplerObj>(std::move(sampler));
} else {
// Mindrecord Sampler
std::shared_ptr<mindrecord::ShardOperator> sampler;
auto create = py::reinterpret_borrow<py::object>(py_sampler.value()).attr("create_for_minddataset");
auto create = py::reinterpret_borrow<py::object>(py_sampler).attr("create_for_minddataset");
sampler = create().cast<std::shared_ptr<mindrecord::ShardOperator>>();
sampler_obj = std::make_shared<PreBuiltSamplerObj>(std::move(sampler));
}
@ -156,10 +156,10 @@ std::shared_ptr<SamplerObj> toSamplerObj(std::optional<py::handle> py_sampler, b
}
// Here we take in a python object, that holds a reference to a C++ object
std::shared_ptr<DatasetCache> toDatasetCache(std::optional<std::shared_ptr<CacheClient>> cc) {
std::shared_ptr<DatasetCache> toDatasetCache(std::shared_ptr<CacheClient> cc) {
if (cc) {
std::shared_ptr<DatasetCache> built_cache;
built_cache = std::make_shared<PreBuiltDatasetCache>(std::move(cc.value()));
built_cache = std::make_shared<PreBuiltDatasetCache>(std::move(cc));
return built_cache;
} else {
// don't need to check here as cache is not enabled.

14
mindspore/ccsrc/minddata/dataset/api/python/pybind_conversion.h
Unescape View File

@ -47,25 +47,25 @@ bool toBool(const py::handle &handle);
std::string toString(const py::handle &handle);
std::set<std::string> toStringSet(const std::optional<py::list> list);
std::set<std::string> toStringSet(const py::list list);
std::map<std::string, int32_t> toStringMap(const std::optional<py::dict> dict);
std::map<std::string, int32_t> toStringMap(const py::dict dict);
std::vector<std::string> toStringVector(const std::optional<py::list> list);
std::vector<std::string> toStringVector(const py::list list);
std::pair<int64_t, int64_t> toIntPair(const std::optional<py::tuple> tuple);
std::pair<int64_t, int64_t> toIntPair(const py::tuple tuple);
std::vector<std::pair<int, int>> toPairVector(const py::list list);
std::vector<std::shared_ptr<TensorOperation>> toTensorOperations(std::optional<py::list> operations);
std::vector<std::shared_ptr<TensorOperation>> toTensorOperations(py::list operations);
std::shared_ptr<TensorOperation> toTensorOperation(py::handle operation);
std::vector<std::shared_ptr<DatasetNode>> toDatasetNode(std::shared_ptr<DatasetNode> self, py::list datasets);
std::shared_ptr<SamplerObj> toSamplerObj(std::optional<py::handle> py_sampler, bool isMindDataset = false);
std::shared_ptr<SamplerObj> toSamplerObj(py::handle py_sampler, bool isMindDataset = false);
std::shared_ptr<DatasetCache> toDatasetCache(std::optional<std::shared_ptr<CacheClient>> cc);
std::shared_ptr<DatasetCache> toDatasetCache(std::shared_ptr<CacheClient> cc);
ShuffleMode toShuffleMode(const int32_t shuffle);

13
mindspore/dataset/engine/datasets.py
Unescape View File

@ -2266,13 +2266,13 @@ class MapDataset(Dataset):
if start_ind != end_ind:
new_ops.append(py_transforms.Compose(operations[start_ind:end_ind]))
operations = new_ops
self.operations = operations
self.operations = replace_none(operations, [])
if input_columns is not None and not isinstance(input_columns, list):
input_columns = [input_columns]
self.input_columns = replace_none(input_columns, [])
if output_columns is not None and not isinstance(output_columns, list):
output_columns = [output_columns]
self.output_columns = replace_none(output_columns, input_columns)
self.output_columns = replace_none(output_columns, self.input_columns)
self.cache = cache
self.column_order = column_order
@ -3025,8 +3025,9 @@ class ImageFolderDataset(MappableDataset):
cc = self.cache.cache_client
else:
cc = None
class_indexing = replace_none(self.class_indexing, {})
return cde.ImageFolderNode(self.dataset_dir, self.decode, self.sampler, self.extensions,
self.class_indexing, cc).SetNumWorkers(self.num_parallel_workers)
class_indexing, cc).SetNumWorkers(self.num_parallel_workers)
def get_args(self):
args = super().get_args()
@ -4043,7 +4044,8 @@ class ManifestDataset(MappableDataset):
cc = self.cache.cache_client
else:
cc = None
return cde.ManifestNode(self.dataset_file, self.usage, self.sampler, self.class_indexing,
class_indexing = replace_none(self.class_indexing, {})
return cde.ManifestNode(self.dataset_file, self.usage, self.sampler, class_indexing,
self.decode, cc).SetNumWorkers(self.num_parallel_workers)
@check_manifestdataset
@ -4701,7 +4703,8 @@ class VOCDataset(MappableDataset):
cc = self.cache.cache_client
else:
cc = None
return cde.VOCNode(self.dataset_dir, self.task, self.usage, self.class_indexing, self.decode,
class_indexing = replace_none(self.class_indexing, {})
return cde.VOCNode(self.dataset_dir, self.task, self.usage, class_indexing, self.decode,
self.sampler, cc).SetNumWorkers(self.num_parallel_workers)
@check_vocdataset

Write Preview
Loading…
Cancel
Save