m53297601
/
mindspore
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Added json fix 2

Browse Source
pull/13365/head
Eric 4 years ago
parent 62d4ef2b11
commit c730df23e8
3 changed files with 32 additions and 32 deletions
Show Stats Download Patch File Download Diff File

26
mindspore/ccsrc/minddata/dataset/api/datasets.cc
Unescape View File

@ -19,7 +19,7 @@
#include <fstream>
#include <unordered_set>
#include <utility>
#include <nlohmann/json.hpp> // we need this before the include to avoid ambiguity in nullptr conversion
#include <nlohmann/json.hpp>
#include "minddata/dataset/core/tensor.h"
#include "minddata/dataset/engine/runtime_context.h"
@ -988,54 +988,54 @@ ManifestDataset::ManifestDataset(const std::vector<char> &dataset_file, const st
MindDataDataset::MindDataDataset(const std::vector<char> &dataset_file,
const std::vector<std::vector<char>> &columns_list,
const std::shared_ptr<Sampler> &sampler, nlohmann::json padded_sample,
const std::shared_ptr<Sampler> &sampler, nlohmann::json *padded_sample,
int64_t num_padded) {
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
auto ds = std::make_shared<MindDataNode>(CharToString(dataset_file), VectorCharToString(columns_list), sampler_obj,
padded_sample, num_padded);
*padded_sample, num_padded);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
}
MindDataDataset::MindDataDataset(const std::vector<char> &dataset_file,
const std::vector<std::vector<char>> &columns_list, Sampler *sampler,
nlohmann::json padded_sample, int64_t num_padded) {
nlohmann::json *padded_sample, int64_t num_padded) {
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
auto ds = std::make_shared<MindDataNode>(CharToString(dataset_file), VectorCharToString(columns_list), sampler_obj,
padded_sample, num_padded);
*padded_sample, num_padded);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
}
MindDataDataset::MindDataDataset(const std::vector<char> &dataset_file,
const std::vector<std::vector<char>> &columns_list,
const std::reference_wrapper<Sampler> sampler, nlohmann::json padded_sample,
const std::reference_wrapper<Sampler> sampler, nlohmann::json *padded_sample,
int64_t num_padded) {
auto sampler_obj = sampler.get().Parse();
auto ds = std::make_shared<MindDataNode>(CharToString(dataset_file), VectorCharToString(columns_list), sampler_obj,
padded_sample, num_padded);
*padded_sample, num_padded);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
}
MindDataDataset::MindDataDataset(const std::vector<std::vector<char>> &dataset_files,
const std::vector<std::vector<char>> &columns_list,
const std::shared_ptr<Sampler> &sampler, nlohmann::json padded_sample,
const std::shared_ptr<Sampler> &sampler, nlohmann::json *padded_sample,
int64_t num_padded) {
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
auto ds = std::make_shared<MindDataNode>(VectorCharToString(dataset_files), VectorCharToString(columns_list),
sampler_obj, padded_sample, num_padded);
sampler_obj, *padded_sample, num_padded);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
}
MindDataDataset::MindDataDataset(const std::vector<std::vector<char>> &dataset_files,
const std::vector<std::vector<char>> &columns_list, Sampler *sampler,
nlohmann::json padded_sample, int64_t num_padded) {
nlohmann::json *padded_sample, int64_t num_padded) {
auto sampler_obj = sampler ? sampler->Parse() : nullptr;
auto ds = std::make_shared<MindDataNode>(VectorCharToString(dataset_files), VectorCharToString(columns_list),
sampler_obj, padded_sample, num_padded);
sampler_obj, *padded_sample, num_padded);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
}
MindDataDataset::MindDataDataset(const std::vector<std::vector<char>> &dataset_files,
const std::vector<std::vector<char>> &columns_list,
const std::reference_wrapper<Sampler> sampler, nlohmann::json padded_sample,
const std::reference_wrapper<Sampler> sampler, nlohmann::json *padded_sample,
int64_t num_padded) {
auto sampler_obj = sampler.get().Parse();
auto ds = std::make_shared<MindDataNode>(VectorCharToString(dataset_files), VectorCharToString(columns_list),
sampler_obj, padded_sample, num_padded);
sampler_obj, *padded_sample, num_padded);
ir_node_ = std::static_pointer_cast<DatasetNode>(ds);
}
#endif

24
mindspore/ccsrc/minddata/dataset/include/datasets.h
Unescape View File

@ -1242,21 +1242,21 @@ inline std::shared_ptr<ManifestDataset> Manifest(const std::string &dataset_file
class MindDataDataset : public Dataset {
public:
explicit MindDataDataset(const std::vector<char> &dataset_file, const std::vector<std::vector<char>> &columns_list,
const std::shared_ptr<Sampler> &sampler, nlohmann::json padded_sample, int64_t num_padded);
const std::shared_ptr<Sampler> &sampler, nlohmann::json *padded_sample, int64_t num_padded);
explicit MindDataDataset(const std::vector<char> &dataset_file, const std::vector<std::vector<char>> &columns_list,
Sampler *sampler, nlohmann::json padded_sample, int64_t num_padded);
Sampler *sampler, nlohmann::json *padded_sample, int64_t num_padded);
explicit MindDataDataset(const std::vector<char> &dataset_file, const std::vector<std::vector<char>> &columns_list,
const std::reference_wrapper<Sampler> sampler, nlohmann::json padded_sample,
const std::reference_wrapper<Sampler> sampler, nlohmann::json *padded_sample,
int64_t num_padded);
explicit MindDataDataset(const std::vector<std::vector<char>> &dataset_files,
const std::vector<std::vector<char>> &columns_list, const std::shared_ptr<Sampler> &sampler,
nlohmann::json padded_sample, int64_t num_padded);
nlohmann::json *padded_sample, int64_t num_padded);
explicit MindDataDataset(const std::vector<std::vector<char>> &dataset_files,
const std::vector<std::vector<char>> &columns_list, Sampler *sampler,
nlohmann::json padded_sample, int64_t num_padded);
nlohmann::json *padded_sample, int64_t num_padded);
explicit MindDataDataset(const std::vector<std::vector<char>> &dataset_files,
const std::vector<std::vector<char>> &columns_list,
const std::reference_wrapper<Sampler> sampler, nlohmann::json padded_sample,
const std::reference_wrapper<Sampler> sampler, nlohmann::json *padded_sample,
int64_t num_padded);
~MindDataDataset() = default;
};
@ -1274,7 +1274,7 @@ class MindDataDataset : public Dataset {
/// \return Shared pointer to the current MindDataDataset
inline std::shared_ptr<MindDataDataset> MindData(
const std::string &dataset_file, const std::vector<std::string> &columns_list = {},
const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(), nlohmann::json padded_sample = nullptr,
const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(), nlohmann::json *padded_sample = nullptr,
int64_t num_padded = 0) {
return std::make_shared<MindDataDataset>(StringToChar(dataset_file), VectorStringToChar(columns_list), sampler,
padded_sample, num_padded);
@ -1291,7 +1291,7 @@ inline std::shared_ptr<MindDataDataset> MindData(
/// \return Shared pointer to the current MindDataDataset
inline std::shared_ptr<MindDataDataset> MindData(const std::string &dataset_file,
const std::vector<std::string> &columns_list, Sampler *sampler,
nlohmann::json padded_sample = nullptr, int64_t num_padded = 0) {
nlohmann::json *padded_sample = nullptr, int64_t num_padded = 0) {
return std::make_shared<MindDataDataset>(StringToChar(dataset_file), VectorStringToChar(columns_list), sampler,
padded_sample, num_padded);
}
@ -1308,7 +1308,7 @@ inline std::shared_ptr<MindDataDataset> MindData(const std::string &dataset_file
inline std::shared_ptr<MindDataDataset> MindData(const std::string &dataset_file,
const std::vector<std::string> &columns_list,
const std::reference_wrapper<Sampler> sampler,
nlohmann::json padded_sample = nullptr, int64_t num_padded = 0) {
nlohmann::json *padded_sample = nullptr, int64_t num_padded = 0) {
return std::make_shared<MindDataDataset>(StringToChar(dataset_file), VectorStringToChar(columns_list), sampler,
padded_sample, num_padded);
}
@ -1325,7 +1325,7 @@ inline std::shared_ptr<MindDataDataset> MindData(const std::string &dataset_file
/// \return Shared pointer to the current MindDataDataset
inline std::shared_ptr<MindDataDataset> MindData(
const std::vector<std::string> &dataset_files, const std::vector<std::string> &columns_list = {},
const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(), nlohmann::json padded_sample = nullptr,
const std::shared_ptr<Sampler> &sampler = std::make_shared<RandomSampler>(), nlohmann::json *padded_sample = nullptr,
int64_t num_padded = 0) {
return std::make_shared<MindDataDataset>(VectorStringToChar(dataset_files), VectorStringToChar(columns_list), sampler,
padded_sample, num_padded);
@ -1341,7 +1341,7 @@ inline std::shared_ptr<MindDataDataset> MindData(
/// \return Shared pointer to the current MindDataDataset
inline std::shared_ptr<MindDataDataset> MindData(const std::vector<std::string> &dataset_files,
const std::vector<std::string> &columns_list, Sampler *sampler,
nlohmann::json padded_sample = nullptr, int64_t num_padded = 0) {
nlohmann::json *padded_sample = nullptr, int64_t num_padded = 0) {
return std::make_shared<MindDataDataset>(VectorStringToChar(dataset_files), VectorStringToChar(columns_list), sampler,
padded_sample, num_padded);
}
@ -1357,7 +1357,7 @@ inline std::shared_ptr<MindDataDataset> MindData(const std::vector<std::string>
inline std::shared_ptr<MindDataDataset> MindData(const std::vector<std::string> &dataset_files,
const std::vector<std::string> &columns_list,
const std::reference_wrapper<Sampler> sampler,
nlohmann::json padded_sample = nullptr, int64_t num_padded = 0) {
nlohmann::json *padded_sample = nullptr, int64_t num_padded = 0) {
return std::make_shared<MindDataDataset>(VectorStringToChar(dataset_files), VectorStringToChar(columns_list), sampler,
padded_sample, num_padded);
}

14
tests/ut/cpp/dataset/c_api_dataset_minddata_test.cc
Unescape View File

@ -289,7 +289,7 @@ TEST_F(MindDataTestPipeline, TestMindDataSuccess7) {
std::string file_path1 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0";
std::vector<std::string> file_list = {file_path1};
std::shared_ptr<Dataset> ds =
MindData(file_list, {"file_name", "label"}, std::make_shared<SequentialSampler>(), pad, 4);
MindData(file_list, {"file_name", "label"}, std::make_shared<SequentialSampler>(), &pad, 4);
EXPECT_NE(ds, nullptr);
// Create a Skip operation on ds, skip original data in mindrecord and get padded samples
@ -341,7 +341,7 @@ TEST_F(MindDataTestPipeline, TestMindDataSuccess8) {
std::string file_path1 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0";
std::vector<std::string> file_list = {file_path1};
std::shared_ptr<Dataset> ds =
MindData(file_list, {"file_name", "label"}, std::make_shared<SequentialSampler>(), pad, 4);
MindData(file_list, {"file_name", "label"}, std::make_shared<SequentialSampler>(), &pad, 4);
EXPECT_NE(ds, nullptr);
std::vector<mindspore::dataset::DataType> types = ToDETypes(ds->GetOutputTypes());
@ -412,13 +412,13 @@ TEST_F(MindDataTestPipeline, TestMindDataSuccess9) {
std::string file_path1 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0";
std::vector<std::string> file_list = {file_path1};
std::shared_ptr<Dataset> ds1 =
MindData(file_list, {"file_name", "label"}, std::make_shared<SequentialSampler>(), pad, 4);
MindData(file_list, {"file_name", "label"}, std::make_shared<SequentialSampler>(), &pad, 4);
EXPECT_NE(ds1, nullptr);
ds1 = ds1->Skip(5);
EXPECT_NE(ds1, nullptr);
std::shared_ptr<Dataset> ds2 =
MindData(file_list, {"file_name", "label"}, std::make_shared<SequentialSampler>(), pad, 4);
MindData(file_list, {"file_name", "label"}, std::make_shared<SequentialSampler>(), &pad, 4);
EXPECT_NE(ds2, nullptr);
ds2 = ds2->Skip(5);
EXPECT_NE(ds2, nullptr);
@ -588,7 +588,7 @@ TEST_F(MindDataTestPipeline, TestMindDataFail4) {
// Create a MindData Dataset
std::string file_path2 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0";
std::shared_ptr<Dataset> ds2 = MindData(file_path2, {"label"}, std::make_shared<RandomSampler>(), pad, -2);
std::shared_ptr<Dataset> ds2 = MindData(file_path2, {"label"}, std::make_shared<RandomSampler>(), &pad, -2);
EXPECT_NE(ds2, nullptr);
// Create an iterator over the result of the above dataset
@ -598,7 +598,7 @@ TEST_F(MindDataTestPipeline, TestMindDataFail4) {
// Create a MindData Dataset
std::string file_path3 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0";
std::shared_ptr<Dataset> ds3 = MindData(file_path3, {}, std::make_shared<RandomSampler>(), pad, 1);
std::shared_ptr<Dataset> ds3 = MindData(file_path3, {}, std::make_shared<RandomSampler>(), &pad, 1);
EXPECT_NE(ds3, nullptr);
// Create an iterator over the result of the above dataset
@ -614,7 +614,7 @@ TEST_F(MindDataTestPipeline, TestMindDataFail4) {
// Create a MindData Dataset
std::string file_path4 = datasets_root_path_ + "/../mindrecord/testMindDataSet/testImageNetData/imagenet.mindrecord0";
std::shared_ptr<Dataset> ds4 =
MindData(file_path4, {"file_name", "label"}, std::make_shared<RandomSampler>(), pad2, 1);
MindData(file_path4, {"file_name", "label"}, std::make_shared<RandomSampler>(), &pad2, 1);
EXPECT_NE(ds4, nullptr);
// Create an iterator over the result of the above dataset

Write Preview
Loading…
Cancel
Save