m53297601
/
mindspore
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

!14389 fix codex warning

Browse Source 
From: @luoyang42
Reviewed-by: @heleiwang,@liucunwei
Signed-off-by: @liucunwei
pull/14389/MERGE
mindspore-ci-bot 4 years ago committed by Gitee
parent 5d7cd47b0b b85a905eaf
commit b6cf16d8aa
11 changed files with 120 additions and 84 deletions
Show Stats Download Patch File Download Diff File

40
mindspore/ccsrc/minddata/dataset/api/datasets.cc
Unescape View File

@ -800,7 +800,7 @@ AlbumDataset::AlbumDataset(const std::vector<char> &dataset_dir, const std::vect
}
AlbumDataset::AlbumDataset(const std::vector<char> &dataset_dir, const std::vector<char> &data_schema,
const std::vector<std::vector<char>> &column_names, bool decode, Sampler *sampler,
const std::vector<std::vector<char>> &column_names, bool decode, const Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache) {
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
auto ds = std::make_shared<AlbumNode>(CharToString(dataset_dir), CharToString(data_schema),
@ -826,8 +826,8 @@ CelebADataset::CelebADataset(const std::vector<char> &dataset_dir, const std::ve
SetCharToString(extensions), cache);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
}
CelebADataset::CelebADataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, Sampler *sampler,
bool decode, const std::set<std::vector<char>> &extensions,
CelebADataset::CelebADataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const Sampler *sampler, bool decode, const std::set<std::vector<char>> &extensions,
const std::shared_ptr<DatasetCache> &cache) {
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
auto ds = std::make_shared<CelebANode>(CharToString(dataset_dir), CharToString(usage), sampler_obj, decode,
@ -850,8 +850,8 @@ Cifar10Dataset::Cifar10Dataset(const std::vector<char> &dataset_dir, const std::
auto ds = std::make_shared<Cifar10Node>(CharToString(dataset_dir), CharToString(usage), sampler_obj, cache);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
}
Cifar10Dataset::Cifar10Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache) {
Cifar10Dataset::Cifar10Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const Sampler *sampler, const std::shared_ptr<DatasetCache> &cache) {
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
auto ds = std::make_shared<Cifar10Node>(CharToString(dataset_dir), CharToString(usage), sampler_obj, cache);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
@ -870,8 +870,8 @@ Cifar100Dataset::Cifar100Dataset(const std::vector<char> &dataset_dir, const std
auto ds = std::make_shared<Cifar100Node>(CharToString(dataset_dir), CharToString(usage), sampler_obj, cache);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
}
Cifar100Dataset::Cifar100Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache) {
Cifar100Dataset::Cifar100Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const Sampler *sampler, const std::shared_ptr<DatasetCache> &cache) {
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
auto ds = std::make_shared<Cifar100Node>(CharToString(dataset_dir), CharToString(usage), sampler_obj, cache);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
@ -901,7 +901,7 @@ CocoDataset::CocoDataset(const std::vector<char> &dataset_dir, const std::vector
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
}
CocoDataset::CocoDataset(const std::vector<char> &dataset_dir, const std::vector<char> &annotation_file,
const std::vector<char> &task, const bool &decode, Sampler *sampler,
const std::vector<char> &task, const bool &decode, const Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache) {
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
auto ds = std::make_shared<CocoNode>(CharToString(dataset_dir), CharToString(annotation_file), CharToString(task),
@ -942,7 +942,7 @@ ImageFolderDataset::ImageFolderDataset(const std::vector<char> &dataset_dir, boo
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
}
ImageFolderDataset::ImageFolderDataset(const std::vector<char> &dataset_dir, bool decode, Sampler *sampler,
ImageFolderDataset::ImageFolderDataset(const std::vector<char> &dataset_dir, bool decode, const Sampler *sampler,
const std::set<std::vector<char>> &extensions,
const std::map<std::vector<char>, int32_t> &class_indexing,
const std::shared_ptr<DatasetCache> &cache) {
@ -981,7 +981,7 @@ ManifestDataset::ManifestDataset(const std::vector<char> &dataset_file, const st
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
}
ManifestDataset::ManifestDataset(const std::vector<char> &dataset_file, const std::vector<char> &usage,
Sampler *sampler, const std::map<std::vector<char>, int32_t> &class_indexing,
const Sampler *sampler, const std::map<std::vector<char>, int32_t> &class_indexing,
bool decode, const std::shared_ptr<DatasetCache> &cache) {
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
auto ds = std::make_shared<ManifestNode>(CharToString(dataset_file), CharToString(usage), sampler_obj,
@ -1000,7 +1000,7 @@ ManifestDataset::ManifestDataset(const std::vector<char> &dataset_file, const st
MindDataDataset::MindDataDataset(const std::vector<char> &dataset_file,
const std::vector<std::vector<char>> &columns_list,
const std::shared_ptr<Sampler> &sampler, nlohmann::json *padded_sample,
const std::shared_ptr<Sampler> &sampler, const nlohmann::json *padded_sample,
int64_t num_padded) {
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
nlohmann::json sample = nullptr;
@ -1012,8 +1012,8 @@ MindDataDataset::MindDataDataset(const std::vector<char> &dataset_file,
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
}
MindDataDataset::MindDataDataset(const std::vector<char> &dataset_file,
const std::vector<std::vector<char>> &columns_list, Sampler *sampler,
nlohmann::json *padded_sample, int64_t num_padded) {
const std::vector<std::vector<char>> &columns_list, const Sampler *sampler,
const nlohmann::json *padded_sample, int64_t num_padded) {
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
nlohmann::json sample = nullptr;
if (padded_sample) {
@ -1025,7 +1025,7 @@ MindDataDataset::MindDataDataset(const std::vector<char> &dataset_file,
}
MindDataDataset::MindDataDataset(const std::vector<char> &dataset_file,
const std::vector<std::vector<char>> &columns_list,
const std::reference_wrapper<Sampler> sampler, nlohmann::json *padded_sample,
const std::reference_wrapper<Sampler> sampler, const nlohmann::json *padded_sample,
int64_t num_padded) {
auto sampler_obj = sampler.get().Parse();
nlohmann::json sample = nullptr;
@ -1039,7 +1039,7 @@ MindDataDataset::MindDataDataset(const std::vector<char> &dataset_file,
}
MindDataDataset::MindDataDataset(const std::vector<std::vector<char>> &dataset_files,
const std::vector<std::vector<char>> &columns_list,
const std::shared_ptr<Sampler> &sampler, nlohmann::json *padded_sample,
const std::shared_ptr<Sampler> &sampler, const nlohmann::json *padded_sample,
int64_t num_padded) {
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
nlohmann::json sample = nullptr;
@ -1052,8 +1052,8 @@ MindDataDataset::MindDataDataset(const std::vector<std::vector<char>> &dataset_f
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
}
MindDataDataset::MindDataDataset(const std::vector<std::vector<char>> &dataset_files,
const std::vector<std::vector<char>> &columns_list, Sampler *sampler,
nlohmann::json *padded_sample, int64_t num_padded) {
const std::vector<std::vector<char>> &columns_list, const Sampler *sampler,
const nlohmann::json *padded_sample, int64_t num_padded) {
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
nlohmann::json sample = nullptr;
if (padded_sample) {
@ -1066,7 +1066,7 @@ MindDataDataset::MindDataDataset(const std::vector<std::vector<char>> &dataset_f
}
MindDataDataset::MindDataDataset(const std::vector<std::vector<char>> &dataset_files,
const std::vector<std::vector<char>> &columns_list,
const std::reference_wrapper<Sampler> sampler, nlohmann::json *padded_sample,
const std::reference_wrapper<Sampler> sampler, const nlohmann::json *padded_sample,
int64_t num_padded) {
auto sampler_obj = sampler.get().Parse();
nlohmann::json sample = nullptr;
@ -1085,7 +1085,7 @@ MnistDataset::MnistDataset(const std::vector<char> &dataset_dir, const std::vect
auto ds = std::make_shared<MnistNode>(CharToString(dataset_dir), CharToString(usage), sampler_obj, cache);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
}
MnistDataset::MnistDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, Sampler *sampler,
MnistDataset::MnistDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, const Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache) {
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
auto ds = std::make_shared<MnistNode>(CharToString(dataset_dir), CharToString(usage), sampler_obj, cache);
@ -1118,7 +1118,7 @@ VOCDataset::VOCDataset(const std::vector<char> &dataset_dir, const std::vector<c
}
VOCDataset::VOCDataset(const std::vector<char> &dataset_dir, const std::vector<char> &task,
const std::vector<char> &usage, const std::map<std::vector<char>, int32_t> &class_indexing,
bool decode, Sampler *sampler, const std::shared_ptr<DatasetCache> &cache) {
bool decode, const Sampler *sampler, const std::shared_ptr<DatasetCache> &cache) {
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
auto ds = std::make_shared<VOCNode>(CharToString(dataset_dir), CharToString(task), CharToString(usage),
MapCharToString(class_indexing), decode, sampler_obj, cache);

14
mindspore/ccsrc/minddata/dataset/api/samplers.cc
Unescape View File

@ -31,7 +31,7 @@ DistributedSampler::DistributedSampler(int64_t num_shards, int64_t shard_id, boo
offset_(offset),
even_dist_(even_dist) {}
std::shared_ptr<SamplerObj> DistributedSampler::Parse() {
std::shared_ptr<SamplerObj> DistributedSampler::Parse() const {
return std::make_shared<DistributedSamplerObj>(num_shards_, shard_id_, shuffle_, num_samples_, seed_, offset_,
even_dist_);
}
@ -40,7 +40,7 @@ std::shared_ptr<SamplerObj> DistributedSampler::Parse() {
PKSampler::PKSampler(int64_t num_val, bool shuffle, int64_t num_samples)
: num_val_(num_val), shuffle_(shuffle), num_samples_(num_samples) {}
std::shared_ptr<SamplerObj> PKSampler::Parse() {
std::shared_ptr<SamplerObj> PKSampler::Parse() const {
return std::make_shared<PKSamplerObj>(num_val_, shuffle_, num_samples_);
}
@ -48,7 +48,7 @@ std::shared_ptr<SamplerObj> PKSampler::Parse() {
RandomSampler::RandomSampler(bool replacement, int64_t num_samples)
: replacement_(replacement), num_samples_(num_samples) {}
std::shared_ptr<SamplerObj> RandomSampler::Parse() {
std::shared_ptr<SamplerObj> RandomSampler::Parse() const {
return std::make_shared<RandomSamplerObj>(replacement_, num_samples_);
}
@ -56,7 +56,7 @@ std::shared_ptr<SamplerObj> RandomSampler::Parse() {
SequentialSampler::SequentialSampler(int64_t start_index, int64_t num_samples)
: start_index_(start_index), num_samples_(num_samples) {}
std::shared_ptr<SamplerObj> SequentialSampler::Parse() {
std::shared_ptr<SamplerObj> SequentialSampler::Parse() const {
return std::make_shared<SequentialSamplerObj>(start_index_, num_samples_);
}
@ -64,7 +64,7 @@ std::shared_ptr<SamplerObj> SequentialSampler::Parse() {
SubsetSampler::SubsetSampler(std::vector<int64_t> indices, int64_t num_samples)
: indices_(indices), num_samples_(num_samples) {}
std::shared_ptr<SamplerObj> SubsetSampler::Parse() {
std::shared_ptr<SamplerObj> SubsetSampler::Parse() const {
return std::make_shared<SubsetSamplerObj>(indices_, num_samples_);
}
@ -72,7 +72,7 @@ std::shared_ptr<SamplerObj> SubsetSampler::Parse() {
SubsetRandomSampler::SubsetRandomSampler(std::vector<int64_t> indices, int64_t num_samples)
: SubsetSampler(indices, num_samples) {}
std::shared_ptr<SamplerObj> SubsetRandomSampler::Parse() {
std::shared_ptr<SamplerObj> SubsetRandomSampler::Parse() const {
return std::make_shared<SubsetRandomSamplerObj>(indices_, num_samples_);
}
@ -80,7 +80,7 @@ std::shared_ptr<SamplerObj> SubsetRandomSampler::Parse() {
WeightedRandomSampler::WeightedRandomSampler(std::vector<double> weights, int64_t num_samples, bool replacement)
: weights_(weights), num_samples_(num_samples), replacement_(replacement) {}
std::shared_ptr<SamplerObj> WeightedRandomSampler::Parse() {
std::shared_ptr<SamplerObj> WeightedRandomSampler::Parse() const {
return std::make_shared<WeightedRandomSamplerObj>(weights_, num_samples_, replacement_);
}

1
mindspore/ccsrc/minddata/dataset/api/text.cc
Unescape View File

@ -298,7 +298,6 @@ struct SentencePieceTokenizer::Data {
: vocab_path_(CharToString(vocab_path)), out_type_(out_type) {}
std::shared_ptr<SentencePieceVocab> vocab_;
std::string vocab_path_;
SPieceTokenizerLoadType load_type_;
SPieceTokenizerOutType out_type_;
};

7
mindspore/ccsrc/minddata/dataset/api/vision.cc
Unescape View File

@ -596,7 +596,8 @@ struct RandomSelectSubpolicy::Data {
std::vector<std::vector<std::pair<std::shared_ptr<TensorOperation>, double>>> policy_;
};
RandomSelectSubpolicy::RandomSelectSubpolicy(std::vector<std::vector<std::pair<TensorTransform *, double>>> policy)
RandomSelectSubpolicy::RandomSelectSubpolicy(
const std::vector<std::vector<std::pair<TensorTransform *, double>>> &policy)
: data_(std::make_shared<Data>()) {
for (int32_t i = 0; i < policy.size(); i++) {
std::vector<std::pair<std::shared_ptr<TensorOperation>, double>> subpolicy;
@ -612,7 +613,7 @@ RandomSelectSubpolicy::RandomSelectSubpolicy(std::vector<std::vector<std::pair<T
}
RandomSelectSubpolicy::RandomSelectSubpolicy(
std::vector<std::vector<std::pair<std::shared_ptr<TensorTransform>, double>>> policy)
const std::vector<std::vector<std::pair<std::shared_ptr<TensorTransform>, double>>> &policy)
: data_(std::make_shared<Data>()) {
for (int32_t i = 0; i < policy.size(); i++) {
std::vector<std::pair<std::shared_ptr<TensorOperation>, double>> subpolicy;
@ -628,7 +629,7 @@ RandomSelectSubpolicy::RandomSelectSubpolicy(
}
RandomSelectSubpolicy::RandomSelectSubpolicy(
std::vector<std::vector<std::pair<std::reference_wrapper<TensorTransform>, double>>> policy)
const std::vector<std::vector<std::pair<std::reference_wrapper<TensorTransform>, double>>> &policy)
: data_(std::make_shared<Data>()) {
for (int32_t i = 0; i < policy.size(); i++) {
std::vector<std::pair<std::shared_ptr<TensorOperation>, double>> subpolicy;

66
mindspore/ccsrc/minddata/dataset/include/datasets.h
Unescape View File

@ -690,7 +690,7 @@ class AlbumDataset : public Dataset {
const std::vector<std::vector<char>> &column_names, bool decode, const std::shared_ptr<Sampler> &sampler,
const std::shared_ptr<DatasetCache> &cache);
AlbumDataset(const std::vector<char> &dataset_dir, const std::vector<char> &data_schema,
const std::vector<std::vector<char>> &column_names, bool decode, Sampler *sampler,
const std::vector<std::vector<char>> &column_names, bool decode, const Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache);
AlbumDataset(const std::vector<char> &dataset_dir, const std::vector<char> &data_schema,
const std::vector<std::vector<char>> &column_names, bool decode,
@ -727,7 +727,8 @@ inline std::shared_ptr<AlbumDataset> Album(const std::string &dataset_dir, const
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
/// \return Shared pointer to the current Dataset
inline std::shared_ptr<AlbumDataset> Album(const std::string &dataset_dir, const std::string &data_schema,
const std::vector<std::string> &column_names, bool decode, Sampler *sampler,
const std::vector<std::string> &column_names, bool decode,
const Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache = nullptr) {
return std::make_shared<AlbumDataset>(StringToChar(dataset_dir), StringToChar(data_schema),
VectorStringToChar(column_names), decode, sampler, cache);
@ -754,7 +755,7 @@ class CelebADataset : public Dataset {
explicit CelebADataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const std::shared_ptr<Sampler> &sampler, bool decode,
const std::set<std::vector<char>> &extensions, const std::shared_ptr<DatasetCache> &cache);
explicit CelebADataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, Sampler *sampler,
explicit CelebADataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, const Sampler *sampler,
bool decode, const std::set<std::vector<char>> &extensions,
const std::shared_ptr<DatasetCache> &cache);
explicit CelebADataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
@ -793,8 +794,9 @@ inline std::shared_ptr<CelebADataset> CelebA(
/// \param[in] extensions Set of file extensions to be included in the dataset (default={}).
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
/// \return Shared pointer to the current Dataset
inline std::shared_ptr<CelebADataset> CelebA(const std::string &dataset_dir, const std::string &usage, Sampler *sampler,
bool decode = false, const std::set<std::string> &extensions = {},
inline std::shared_ptr<CelebADataset> CelebA(const std::string &dataset_dir, const std::string &usage,
const Sampler *sampler, bool decode = false,
const std::set<std::string> &extensions = {},
const std::shared_ptr<DatasetCache> &cache = nullptr) {
return std::make_shared<CelebADataset>(StringToChar(dataset_dir), StringToChar(usage), sampler, decode,
SetStringToChar(extensions), cache);
@ -822,7 +824,7 @@ class Cifar10Dataset : public Dataset {
public:
explicit Cifar10Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache);
explicit Cifar10Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, Sampler *sampler,
explicit Cifar10Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, const Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache);
explicit Cifar10Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const std::reference_wrapper<Sampler> sampler, const std::shared_ptr<DatasetCache> &cache);
@ -853,7 +855,8 @@ inline std::shared_ptr<Cifar10Dataset> Cifar10(
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
/// \return Shared pointer to the current Dataset
inline std::shared_ptr<Cifar10Dataset> Cifar10(const std::string &dataset_dir, const std::string &usage,
Sampler *sampler, const std::shared_ptr<DatasetCache> &cache = nullptr) {
const Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache = nullptr) {
return std::make_shared<Cifar10Dataset>(StringToChar(dataset_dir), StringToChar(usage), sampler, cache);
}
@ -874,7 +877,7 @@ class Cifar100Dataset : public Dataset {
public:
explicit Cifar100Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache);
explicit Cifar100Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, Sampler *sampler,
explicit Cifar100Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, const Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache);
explicit Cifar100Dataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const std::reference_wrapper<Sampler> sampler, const std::shared_ptr<DatasetCache> &cache);
@ -905,7 +908,7 @@ inline std::shared_ptr<Cifar100Dataset> Cifar100(
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
/// \return Shared pointer to the current Dataset
inline std::shared_ptr<Cifar100Dataset> Cifar100(const std::string &dataset_dir, const std::string &usage,
Sampler *sampler,
const Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache = nullptr) {
return std::make_shared<Cifar100Dataset>(StringToChar(dataset_dir), StringToChar(usage), sampler, cache);
}
@ -964,7 +967,7 @@ class CocoDataset : public Dataset {
const std::vector<char> &task, const bool &decode, const std::shared_ptr<Sampler> &sampler,
const std::shared_ptr<DatasetCache> &cache);
CocoDataset(const std::vector<char> &dataset_dir, const std::vector<char> &annotation_file,
const std::vector<char> &task, const bool &decode, Sampler *sampler,
const std::vector<char> &task, const bool &decode, const Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache);
CocoDataset(const std::vector<char> &dataset_dir, const std::vector<char> &annotation_file,
const std::vector<char> &task, const bool &decode, const std::reference_wrapper<Sampler> sampler,
@ -1015,7 +1018,7 @@ inline std::shared_ptr<CocoDataset> Coco(const std::string &dataset_dir, const s
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
/// \return Shared pointer to the current Dataset
inline std::shared_ptr<CocoDataset> Coco(const std::string &dataset_dir, const std::string &annotation_file,
const std::string &task, const bool &decode, Sampler *sampler,
const std::string &task, const bool &decode, const Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache = nullptr) {
return std::make_shared<CocoDataset>(StringToChar(dataset_dir), StringToChar(annotation_file), StringToChar(task),
decode, sampler, cache);
@ -1091,7 +1094,7 @@ class ImageFolderDataset : public Dataset {
const std::shared_ptr<Sampler> &sampler, const std::set<std::vector<char>> &extensions,
const std::map<std::vector<char>, int32_t> &class_indexing,
const std::shared_ptr<DatasetCache> &cache);
explicit ImageFolderDataset(const std::vector<char> &dataset_dir, bool decode, Sampler *sampler,
explicit ImageFolderDataset(const std::vector<char> &dataset_dir, bool decode, const Sampler *sampler,
const std::set<std::vector<char>> &extensions,
const std::map<std::vector<char>, int32_t> &class_indexing,
const std::shared_ptr<DatasetCache> &cache);
@ -1136,7 +1139,8 @@ inline std::shared_ptr<ImageFolderDataset> ImageFolder(
/// \param[in] class_indexing a class name to label map
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
/// \return Shared pointer to the current ImageFolderDataset
inline std::shared_ptr<ImageFolderDataset> ImageFolder(const std::string &dataset_dir, bool decode, Sampler *sampler,
inline std::shared_ptr<ImageFolderDataset> ImageFolder(const std::string &dataset_dir, bool decode,
const Sampler *sampler,
const std::set<std::string> &extensions = {},
const std::map<std::string, int32_t> &class_indexing = {},
const std::shared_ptr<DatasetCache> &cache = nullptr) {
@ -1170,9 +1174,9 @@ class ManifestDataset : public Dataset {
const std::shared_ptr<Sampler> &sampler,
const std::map<std::vector<char>, int32_t> &class_indexing, bool decode,
const std::shared_ptr<DatasetCache> &cache);
explicit ManifestDataset(const std::vector<char> &dataset_file, const std::vector<char> &usage, Sampler *sampler,
const std::map<std::vector<char>, int32_t> &class_indexing, bool decode,
const std::shared_ptr<DatasetCache> &cache);
explicit ManifestDataset(const std::vector<char> &dataset_file, const std::vector<char> &usage,
const Sampler *sampler, const std::map<std::vector<char>, int32_t> &class_indexing,
bool decode, const std::shared_ptr<DatasetCache> &cache);
explicit ManifestDataset(const std::vector<char> &dataset_file, const std::vector<char> &usage,
const std::reference_wrapper<Sampler> sampler,
const std::map<std::vector<char>, int32_t> &class_indexing, bool decode,
@ -1212,7 +1216,7 @@ inline std::shared_ptr<ManifestDataset> Manifest(
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
/// \return Shared pointer to the current ManifestDataset
inline std::shared_ptr<ManifestDataset> Manifest(const std::string &dataset_file, const std::string &usage,
Sampler *sampler,
const Sampler *sampler,
const std::map<std::string, int32_t> &class_indexing = {},
bool decode = false,
const std::shared_ptr<DatasetCache> &cache = nullptr) {
@ -1242,21 +1246,22 @@ inline std::shared_ptr<ManifestDataset> Manifest(const std::string &dataset_file
class MindDataDataset : public Dataset {
public:
explicit MindDataDataset(const std::vector<char> &dataset_file, const std::vector<std::vector<char>> &columns_list,
const std::shared_ptr<Sampler> &sampler, nlohmann::json *padded_sample, int64_t num_padded);
const std::shared_ptr<Sampler> &sampler, const nlohmann::json *padded_sample,
int64_t num_padded);
explicit MindDataDataset(const std::vector<char> &dataset_file, const std::vector<std::vector<char>> &columns_list,
Sampler *sampler, nlohmann::json *padded_sample, int64_t num_padded);
const Sampler *sampler, const nlohmann::json *padded_sample, int64_t num_padded);
explicit MindDataDataset(const std::vector<char> &dataset_file, const std::vector<std::vector<char>> &columns_list,
const std::reference_wrapper<Sampler> sampler, nlohmann::json *padded_sample,
const std::reference_wrapper<Sampler> sampler, const nlohmann::json *padded_sample,
int64_t num_padded);
explicit MindDataDataset(const std::vector<std::vector<char>> &dataset_files,
const std::vector<std::vector<char>> &columns_list, const std::shared_ptr<Sampler> &sampler,
nlohmann::json *padded_sample, int64_t num_padded);
const nlohmann::json *padded_sample, int64_t num_padded);
explicit MindDataDataset(const std::vector<std::vector<char>> &dataset_files,
const std::vector<std::vector<char>> &columns_list, Sampler *sampler,
nlohmann::json *padded_sample, int64_t num_padded);
const std::vector<std::vector<char>> &columns_list, const Sampler *sampler,
const nlohmann::json *padded_sample, int64_t num_padded);
explicit MindDataDataset(const std::vector<std::vector<char>> &dataset_files,
const std::vector<std::vector<char>> &columns_list,
const std::reference_wrapper<Sampler> sampler, nlohmann::json *padded_sample,
const std::reference_wrapper<Sampler> sampler, const nlohmann::json *padded_sample,
int64_t num_padded);
~MindDataDataset() = default;
};
@ -1290,7 +1295,7 @@ inline std::shared_ptr<MindDataDataset> MindData(
/// \param[in] num_padded Number of padding samples. Dataset size plus num_padded should be divisible by num_shards.
/// \return Shared pointer to the current MindDataDataset
inline std::shared_ptr<MindDataDataset> MindData(const std::string &dataset_file,
const std::vector<std::string> &columns_list, Sampler *sampler,
const std::vector<std::string> &columns_list, const Sampler *sampler,
nlohmann::json *padded_sample = nullptr, int64_t num_padded = 0) {
return std::make_shared<MindDataDataset>(StringToChar(dataset_file), VectorStringToChar(columns_list), sampler,
padded_sample, num_padded);
@ -1340,7 +1345,7 @@ inline std::shared_ptr<MindDataDataset> MindData(
/// \param[in] num_padded Number of padding samples. Dataset size plus num_padded should be divisible by num_shards.
/// \return Shared pointer to the current MindDataDataset
inline std::shared_ptr<MindDataDataset> MindData(const std::vector<std::string> &dataset_files,
const std::vector<std::string> &columns_list, Sampler *sampler,
const std::vector<std::string> &columns_list, const Sampler *sampler,
nlohmann::json *padded_sample = nullptr, int64_t num_padded = 0) {
return std::make_shared<MindDataDataset>(VectorStringToChar(dataset_files), VectorStringToChar(columns_list), sampler,
padded_sample, num_padded);
@ -1366,7 +1371,7 @@ class MnistDataset : public Dataset {
public:
explicit MnistDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache);
explicit MnistDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, Sampler *sampler,
explicit MnistDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, const Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache);
explicit MnistDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const std::reference_wrapper<Sampler> sampler, const std::shared_ptr<DatasetCache> &cache);
@ -1395,7 +1400,8 @@ inline std::shared_ptr<MnistDataset> Mnist(const std::string &dataset_dir, const
/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset.
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
/// \return Shared pointer to the current MnistDataset
inline std::shared_ptr<MnistDataset> Mnist(const std::string &dataset_dir, const std::string &usage, Sampler *sampler,
inline std::shared_ptr<MnistDataset> Mnist(const std::string &dataset_dir, const std::string &usage,
const Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache = nullptr) {
return std::make_shared<MnistDataset>(StringToChar(dataset_dir), StringToChar(usage), sampler, cache);
}
@ -1560,7 +1566,7 @@ class VOCDataset : public Dataset {
bool decode, const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache);
explicit VOCDataset(const std::vector<char> &dataset_dir, const std::vector<char> &task,
const std::vector<char> &usage, const std::map<std::vector<char>, int32_t> &class_indexing,
bool decode, Sampler *sampler, const std::shared_ptr<DatasetCache> &cache);
bool decode, const Sampler *sampler, const std::shared_ptr<DatasetCache> &cache);
explicit VOCDataset(const std::vector<char> &dataset_dir, const std::vector<char> &task,
const std::vector<char> &usage, const std::map<std::vector<char>, int32_t> &class_indexing,
bool decode, const std::reference_wrapper<Sampler> sampler,
@ -1607,7 +1613,7 @@ inline std::shared_ptr<VOCDataset> VOC(const std::string &dataset_dir, const std
/// \return Shared pointer to the current Dataset
inline std::shared_ptr<VOCDataset> VOC(const std::string &dataset_dir, const std::string &task,
const std::string &usage, const std::map<std::string, int32_t> &class_indexing,
bool decode, Sampler *sampler,
bool decode, const Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache = nullptr) {
return std::make_shared<VOCDataset>(StringToChar(dataset_dir), StringToChar(task), StringToChar(usage),
MapStringToChar(class_indexing), decode, sampler, cache);

16
mindspore/ccsrc/minddata/dataset/include/samplers.h
Unescape View File

@ -61,7 +61,7 @@ class Sampler : std::enable_shared_from_this<Sampler> {
protected:
/// \brief Pure virtual function to convert a Sampler class into an IR Sampler object.
/// \return shared pointer to the newly created TensorOperation.
virtual std::shared_ptr<SamplerObj> Parse() = 0;
virtual std::shared_ptr<SamplerObj> Parse() const = 0;
std::vector<std::shared_ptr<Sampler>> children_;
};
@ -89,7 +89,7 @@ class DistributedSampler final : public Sampler {
protected:
/// \brief Function to convert a Sampler into an IR SamplerObj.
/// \return shared pointer to the newly created SamplerObj.
std::shared_ptr<SamplerObj> Parse() override;
std::shared_ptr<SamplerObj> Parse() const override;
private:
int64_t num_shards_;
@ -120,7 +120,7 @@ class PKSampler final : public Sampler {
protected:
/// \brief Function to convert a Sampler into an IR SamplerObj.
/// \return shared pointer to the newly created SamplerObj.
std::shared_ptr<SamplerObj> Parse() override;
std::shared_ptr<SamplerObj> Parse() const override;
private:
int64_t num_val_;
@ -145,7 +145,7 @@ class RandomSampler final : public Sampler {
protected:
/// \brief Function to convert a Sampler into an IR SamplerObj.
/// \return shared pointer to the newly created SamplerObj.
std::shared_ptr<SamplerObj> Parse() override;
std::shared_ptr<SamplerObj> Parse() const override;
private:
bool replacement_;
@ -169,7 +169,7 @@ class SequentialSampler final : public Sampler {
protected:
/// \brief Function to convert a Sampler into an IR SamplerObj.
/// \return shared pointer to the newly created SamplerObj.
std::shared_ptr<SamplerObj> Parse() override;
std::shared_ptr<SamplerObj> Parse() const override;
private:
int64_t start_index_;
@ -193,7 +193,7 @@ class SubsetSampler : public Sampler {
protected:
/// \brief Function to convert a Sampler into an IR SamplerObj.
/// \return shared pointer to the newly created SamplerObj.
std::shared_ptr<SamplerObj> Parse() override;
std::shared_ptr<SamplerObj> Parse() const override;
std::vector<int64_t> indices_;
int64_t num_samples_;
@ -216,7 +216,7 @@ class SubsetRandomSampler final : public SubsetSampler {
protected:
/// \brief Function to convert a Sampler into an IR SamplerObj.
/// \return shared pointer to the newly created SamplerObj.
std::shared_ptr<SamplerObj> Parse() override;
std::shared_ptr<SamplerObj> Parse() const override;
};
/// \brief A class to represent a Weighted Random Sampler in the data pipeline.
@ -238,7 +238,7 @@ class WeightedRandomSampler final : public Sampler {
protected:
/// \brief Function to convert a Sampler into an IR SamplerObj.
/// \return shared pointer to the newly created SamplerObj.
std::shared_ptr<SamplerObj> Parse() override;
std::shared_ptr<SamplerObj> Parse() const override;
private:
std::vector<double> weights_;

7
mindspore/ccsrc/minddata/dataset/include/vision.h
Unescape View File

@ -648,16 +648,17 @@ class RandomSelectSubpolicy final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] policy Vector of sub-policies to choose from, in which the TensorTransform objects are raw pointers
explicit RandomSelectSubpolicy(std::vector<std::vector<std::pair<TensorTransform *, double>>> policy);
explicit RandomSelectSubpolicy(const std::vector<std::vector<std::pair<TensorTransform *, double>>> &policy);
/// \brief Constructor.
/// \param[in] policy Vector of sub-policies to choose from, in which the TensorTransform objects are shared pointers
explicit RandomSelectSubpolicy(std::vector<std::vector<std::pair<std::shared_ptr<TensorTransform>, double>>> policy);
explicit RandomSelectSubpolicy(
const std::vector<std::vector<std::pair<std::shared_ptr<TensorTransform>, double>>> &policy);
/// \brief Constructor.
/// \param[in] policy Vector of sub-policies to choose from, in which the TensorTransform objects are object pointers
explicit RandomSelectSubpolicy(
std::vector<std::vector<std::pair<std::reference_wrapper<TensorTransform>, double>>> policy);
const std::vector<std::vector<std::pair<std::reference_wrapper<TensorTransform>, double>>> &policy);
/// \brief Destructor.
~RandomSelectSubpolicy() = default;

10
mindspore/ccsrc/minddata/dataset/liteapi/include/datasets.h
Unescape View File

@ -456,7 +456,7 @@ class AlbumDataset : public Dataset {
const std::vector<std::vector<char>> &column_names, bool decode, const std::shared_ptr<Sampler> &sampler,
const std::shared_ptr<DatasetCache> &cache);
AlbumDataset(const std::vector<char> &dataset_dir, const std::vector<char> &data_schema,
const std::vector<std::vector<char>> &column_names, bool decode, Sampler *sampler,
const std::vector<std::vector<char>> &column_names, bool decode, const Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache);
AlbumDataset(const std::vector<char> &dataset_dir, const std::vector<char> &data_schema,
const std::vector<std::vector<char>> &column_names, bool decode,
@ -493,7 +493,8 @@ inline std::shared_ptr<AlbumDataset> Album(const std::string &dataset_dir, const
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
/// \return Shared pointer to the current Dataset
inline std::shared_ptr<AlbumDataset> Album(const std::string &dataset_dir, const std::string &data_schema,
const std::vector<std::string> &column_names, bool decode, Sampler *sampler,
const std::vector<std::string> &column_names, bool decode,
const Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache = nullptr) {
return std::make_shared<AlbumDataset>(StringToChar(dataset_dir), StringToChar(data_schema),
VectorStringToChar(column_names), decode, sampler, cache);
@ -519,7 +520,7 @@ class MnistDataset : public Dataset {
public:
explicit MnistDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const std::shared_ptr<Sampler> &sampler, const std::shared_ptr<DatasetCache> &cache);
explicit MnistDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, Sampler *sampler,
explicit MnistDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage, const Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache);
explicit MnistDataset(const std::vector<char> &dataset_dir, const std::vector<char> &usage,
const std::reference_wrapper<Sampler> sampler, const std::shared_ptr<DatasetCache> &cache);
@ -548,7 +549,8 @@ inline std::shared_ptr<MnistDataset> Mnist(const std::string &dataset_dir, const
/// \param[in] sampler Raw pointer to a sampler object used to choose samples from the dataset.
/// \param[in] cache Tensor cache to use. (default=nullptr which means no cache is used).
/// \return Shared pointer to the current MnistDataset
inline std::shared_ptr<MnistDataset> Mnist(const std::string &dataset_dir, const std::string &usage, Sampler *sampler,
inline std::shared_ptr<MnistDataset> Mnist(const std::string &dataset_dir, const std::string &usage,
const Sampler *sampler,
const std::shared_ptr<DatasetCache> &cache = nullptr) {
return std::make_shared<MnistDataset>(StringToChar(dataset_dir), StringToChar(usage), sampler, cache);
}

16
mindspore/ccsrc/minddata/dataset/liteapi/include/samplers.h
Unescape View File

@ -61,7 +61,7 @@ class Sampler : std::enable_shared_from_this<Sampler> {
protected:
/// \brief Pure virtual function to convert a Sampler class into an IR Sampler object.
/// \return shared pointer to the newly created TensorOperation.
virtual std::shared_ptr<SamplerObj> Parse() = 0;
virtual std::shared_ptr<SamplerObj> Parse() const = 0;
std::vector<std::shared_ptr<Sampler>> children_;
};
@ -89,7 +89,7 @@ class DistributedSampler final : public Sampler {
protected:
/// \brief Function to convert a Sampler into an IR SamplerObj.
/// \return shared pointer to the newly created SamplerObj.
std::shared_ptr<SamplerObj> Parse() override;
std::shared_ptr<SamplerObj> Parse() const override;
private:
int64_t num_shards_;
@ -120,7 +120,7 @@ class PKSampler final : public Sampler {
protected:
/// \brief Function to convert a Sampler into an IR SamplerObj.
/// \return shared pointer to the newly created SamplerObj.
std::shared_ptr<SamplerObj> Parse() override;
std::shared_ptr<SamplerObj> Parse() const override;
private:
int64_t num_val_;
@ -145,7 +145,7 @@ class RandomSampler final : public Sampler {
protected:
/// \brief Function to convert a Sampler into an IR SamplerObj.
/// \return shared pointer to the newly created SamplerObj.
std::shared_ptr<SamplerObj> Parse() override;
std::shared_ptr<SamplerObj> Parse() const override;
private:
bool replacement_;
@ -169,7 +169,7 @@ class SequentialSampler final : public Sampler {
protected:
/// \brief Function to convert a Sampler into an IR SamplerObj.
/// \return shared pointer to the newly created SamplerObj.
std::shared_ptr<SamplerObj> Parse() override;
std::shared_ptr<SamplerObj> Parse() const override;
private:
int64_t start_index_;
@ -193,7 +193,7 @@ class SubsetSampler : public Sampler {
protected:
/// \brief Function to convert a Sampler into an IR SamplerObj.
/// \return shared pointer to the newly created SamplerObj.
std::shared_ptr<SamplerObj> Parse() override;
std::shared_ptr<SamplerObj> Parse() const override;
std::vector<int64_t> indices_;
int64_t num_samples_;
@ -216,7 +216,7 @@ class SubsetRandomSampler final : public SubsetSampler {
protected:
/// \brief Function to convert a Sampler into an IR SamplerObj.
/// \return shared pointer to the newly created SamplerObj.
std::shared_ptr<SamplerObj> Parse() override;
std::shared_ptr<SamplerObj> Parse() const override;
};
/// \brief A class to represent a Weighted Random Sampler in the data pipeline.
@ -238,7 +238,7 @@ class WeightedRandomSampler final : public Sampler {
protected:
/// \brief Function to convert a Sampler into an IR SamplerObj.
/// \return shared pointer to the newly created SamplerObj.
std::shared_ptr<SamplerObj> Parse() override;
std::shared_ptr<SamplerObj> Parse() const override;
private:
std::vector<double> weights_;

0
model_zoo/research/cv/centernet_det/scripts/ascend_distributed_launcher/__init__.py
Unescape View File

27
tests/ut/cpp/dataset/c_api_samplers_test.cc
Unescape View File

@ -207,6 +207,33 @@ TEST_F(MindDataTestPipeline, TestDistributedSamplerSuccess3) {
iter->Stop();
}
TEST_F(MindDataTestPipeline, TestDistributedSamplerSuccess4) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDistributedSamplerSuccess4.";
// Test pointer of distributed_sampler
SequentialSampler sampler = SequentialSampler(0, 4);
// Create an ImageFolder Dataset
std::string folder_path = datasets_root_path_ + "/testVOC2012_2";
std::shared_ptr<Dataset> ds = VOC(folder_path, "Detection", "train", {}, false, &sampler);
EXPECT_NE(ds, nullptr);
// Iterate the dataset and get each row
std::shared_ptr<Iterator> iter = ds->CreateIterator();
EXPECT_NE(iter, nullptr);
std::unordered_map<std::string, mindspore::MSTensor> row;
iter->GetNextRow(&row);
uint64_t i = 0;
while (row.size() != 0) {
i++;
auto label = row["label"];
iter->GetNextRow(&row);
}
EXPECT_EQ(i, 4);
iter->Stop();
}
TEST_F(MindDataTestPipeline, TestDistributedSamplerFail1) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestDistributedSamplerFail1.";
// Test basic setting of distributed_sampler

Write Preview
Loading…
Cancel
Save