m53297601
/
mindspore
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

!11563 Change num_parallel_worker in UT

Browse Source 
From: @hfarahat
Reviewed-by: @nsyca,@robingrosman
Signed-off-by: @robingrosman
pull/11563/MERGE
mindspore-ci-bot 4 years ago committed by Gitee
parent b1a44e8875 715d70f397
commit 8d137c9840
11 changed files with 32 additions and 75 deletions
Show Stats Download Patch File Download Diff File

2
tests/ut/cpp/dataset/c_api_repeat_test.cc
Unescape View File

@ -28,7 +28,7 @@ TEST_F(MindDataTestPipeline, TestRepeatSetNumWorkers) {
std::string file_path = datasets_root_path_ + "/testTFTestAllTypes/test.data"; std::string file_path = datasets_root_path_ + "/testTFTestAllTypes/test.data";
std::shared_ptr<Dataset> ds = TFRecord({file_path}); std::shared_ptr<Dataset> ds = TFRecord({file_path});
ds = ds->SetNumWorkers(16); ds = ds->SetNumWorkers(8);
ds = ds->Repeat(32); ds = ds->Repeat(32);
// Create an iterator over the result of the above dataset // Create an iterator over the result of the above dataset

2
tests/ut/cpp/dataset/client_config_test.cc
Unescape View File

@ -77,7 +77,7 @@ TEST_F(MindDataTestClientConfig, TestClientConfig1) {
TEST_F(MindDataTestClientConfig, TestClientConfig2) { TEST_F(MindDataTestClientConfig, TestClientConfig2) {
std::shared_ptr<ConfigManager> my_conf = GlobalContext::config_manager(); std::shared_ptr<ConfigManager> my_conf = GlobalContext::config_manager();
my_conf->set_num_parallel_workers(16); my_conf->set_num_parallel_workers(8);
Status rc; Status rc;

1
tests/ut/cpp/dataset/clue_op_test.cc
Unescape View File

@ -51,7 +51,6 @@ TEST_F(MindDataTestCLUEOp, TestCLUEBasic) {
ClueOp::Builder builder; ClueOp::Builder builder;
builder.SetClueFilesList({dataset_path}) builder.SetClueFilesList({dataset_path})
.SetRowsPerBuffer(16) .SetRowsPerBuffer(16)
.SetNumWorkers(16)
.SetOpConnectorSize(2) .SetOpConnectorSize(2)
.SetColsKeyMap(key_map); .SetColsKeyMap(key_map);

10
tests/ut/cpp/dataset/concat_op_test.cc
Unescape View File

@ -51,10 +51,7 @@ TEST_F(MindDataTestConcatOp, TestConcatProject) {
// TFReaderOp1 // TFReaderOp1
std::shared_ptr<TFReaderOp> my_tfreader_op1; std::shared_ptr<TFReaderOp> my_tfreader_op1;
TFReaderOp::Builder builder1; TFReaderOp::Builder builder1;
builder1.SetDatasetFilesList({dataset_path}) builder1.SetDatasetFilesList({dataset_path}).SetRowsPerBuffer(16).SetWorkerConnectorSize(16);
.SetRowsPerBuffer(16)
.SetWorkerConnectorSize(16)
.SetNumWorkers(16);
std::unique_ptr<DataSchema> schema1 = std::make_unique<DataSchema>(); std::unique_ptr<DataSchema> schema1 = std::make_unique<DataSchema>();
schema1->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {}); schema1->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {});
builder1.SetDataSchema(std::move(schema1)); builder1.SetDataSchema(std::move(schema1));
@ -66,10 +63,7 @@ TEST_F(MindDataTestConcatOp, TestConcatProject) {
// TFReaderOp2 // TFReaderOp2
std::shared_ptr<TFReaderOp> my_tfreader_op2; std::shared_ptr<TFReaderOp> my_tfreader_op2;
TFReaderOp::Builder builder2; TFReaderOp::Builder builder2;
builder2.SetDatasetFilesList({dataset_path}) builder2.SetDatasetFilesList({dataset_path}).SetRowsPerBuffer(16).SetWorkerConnectorSize(16);
.SetRowsPerBuffer(16)
.SetWorkerConnectorSize(16)
.SetNumWorkers(16);
std::unique_ptr<DataSchema> schema2 = std::make_unique<DataSchema>(); std::unique_ptr<DataSchema> schema2 = std::make_unique<DataSchema>();
schema2->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {}); schema2->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {});
builder2.SetDataSchema(std::move(schema2)); builder2.SetDataSchema(std::move(schema2));

1
tests/ut/cpp/dataset/csv_op_test.cc
Unescape View File

@ -53,7 +53,6 @@ TEST_F(MindDataTestCSVOp, TestCSVBasic) {
CsvOp::Builder builder; CsvOp::Builder builder;
builder.SetCsvFilesList({dataset_path}) builder.SetCsvFilesList({dataset_path})
.SetRowsPerBuffer(16) .SetRowsPerBuffer(16)
.SetNumWorkers(16)
.SetShuffleFiles(false) .SetShuffleFiles(false)
.SetOpConnectorSize(2) .SetOpConnectorSize(2)
.SetFieldDelim(',') .SetFieldDelim(',')

2
tests/ut/cpp/dataset/ir_tensor_op_fusion_pass_test.cc
Unescape View File

@ -37,7 +37,6 @@ TEST_F(MindDataTestTensorOpFusionPass, RandomCropDecodeResizeDisabled) {
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, SequentialSampler(0, 11)); std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, SequentialSampler(0, 11));
ds = ds->SetNumWorkers(16);
// Create objects for the tensor ops // Create objects for the tensor ops
std::shared_ptr<TensorOperation> decode = vision::Decode(); std::shared_ptr<TensorOperation> decode = vision::Decode();
@ -69,7 +68,6 @@ TEST_F(MindDataTestTensorOpFusionPass, RandomCropDecodeResizeEnabled) {
std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::string folder_path = datasets_root_path_ + "/testPK/data/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, SequentialSampler(0, 11)); std::shared_ptr<Dataset> ds = ImageFolder(folder_path, false, SequentialSampler(0, 11));
ds = ds->SetNumWorkers(16);
// Create objects for the tensor ops // Create objects for the tensor ops
std::shared_ptr<TensorOperation> decode = vision::Decode(); std::shared_ptr<TensorOperation> decode = vision::Decode();

5
tests/ut/cpp/dataset/project_op_test.cc
Unescape View File

@ -41,10 +41,7 @@ TEST_F(MindDataTestProjectOp, TestProjectProject) {
std::shared_ptr<TFReaderOp> my_tfreader_op; std::shared_ptr<TFReaderOp> my_tfreader_op;
TFReaderOp::Builder builder; TFReaderOp::Builder builder;
builder.SetDatasetFilesList({dataset_path}) builder.SetDatasetFilesList({dataset_path}).SetRowsPerBuffer(16).SetWorkerConnectorSize(16);
.SetRowsPerBuffer(16)
.SetWorkerConnectorSize(16)
.SetNumWorkers(16);
std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>(); std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>();
schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {}); schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {});
builder.SetDataSchema(std::move(schema)); builder.SetDataSchema(std::move(schema));

5
tests/ut/cpp/dataset/skip_op_test.cc
Unescape View File

@ -35,10 +35,7 @@ TEST_F(MindDataTestSkipOp, TestSkipOpFuntions) {
std::shared_ptr<TFReaderOp> my_tfreader_op; std::shared_ptr<TFReaderOp> my_tfreader_op;
TFReaderOp::Builder builder; TFReaderOp::Builder builder;
builder.SetDatasetFilesList({dataset_path}) builder.SetDatasetFilesList({dataset_path}).SetRowsPerBuffer(16).SetWorkerConnectorSize(16);
.SetRowsPerBuffer(16)
.SetWorkerConnectorSize(16)
.SetNumWorkers(16);
std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>(); std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>();
schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {}); schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {});
builder.SetDataSchema(std::move(schema)); builder.SetDataSchema(std::move(schema));

5
tests/ut/cpp/dataset/take_op_test.cc
Unescape View File

@ -42,10 +42,7 @@ TEST_F(MindDataTestTakeOp, TestTakeProject) {
// TFReaderOp // TFReaderOp
std::shared_ptr<TFReaderOp> my_tfreader_op; std::shared_ptr<TFReaderOp> my_tfreader_op;
TFReaderOp::Builder builder; TFReaderOp::Builder builder;
builder.SetDatasetFilesList({dataset_path}) builder.SetDatasetFilesList({dataset_path}).SetRowsPerBuffer(16).SetWorkerConnectorSize(16);
.SetRowsPerBuffer(16)
.SetWorkerConnectorSize(16)
.SetNumWorkers(16);
std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>(); std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>();
schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {}); schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {});
builder.SetDataSchema(std::move(schema)); builder.SetDataSchema(std::move(schema));

10
tests/ut/cpp/dataset/text_file_op_test.cc
Unescape View File

@ -45,10 +45,7 @@ TEST_F(MindDataTestTextFileOp, TestTextFileBasic) {
std::shared_ptr<TextFileOp> op; std::shared_ptr<TextFileOp> op;
TextFileOp::Builder builder; TextFileOp::Builder builder;
builder.SetTextFilesList({dataset_path}) builder.SetTextFilesList({dataset_path}).SetRowsPerBuffer(16).SetOpConnectorSize(2);
.SetRowsPerBuffer(16)
.SetNumWorkers(16)
.SetOpConnectorSize(2);
Status rc = builder.Build(&op); Status rc = builder.Build(&op);
ASSERT_TRUE(rc.IsOk()); ASSERT_TRUE(rc.IsOk());
@ -97,10 +94,7 @@ TEST_F(MindDataTestTextFileOp, TestTextFileFileNotExist) {
std::shared_ptr<TextFileOp> op; std::shared_ptr<TextFileOp> op;
TextFileOp::Builder builder; TextFileOp::Builder builder;
builder.SetTextFilesList({dataset_path}) builder.SetTextFilesList({dataset_path}).SetRowsPerBuffer(16).SetOpConnectorSize(2);
.SetRowsPerBuffer(16)
.SetNumWorkers(16)
.SetOpConnectorSize(2);
Status rc = builder.Build(&op); Status rc = builder.Build(&op);
ASSERT_TRUE(rc.IsOk()); ASSERT_TRUE(rc.IsOk());

44
tests/ut/cpp/dataset/tfReader_op_test.cc
Unescape View File

@ -44,9 +44,7 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderBasic1) {
std::shared_ptr<TFReaderOp> my_tfreader_op; std::shared_ptr<TFReaderOp> my_tfreader_op;
TFReaderOp::Builder builder; TFReaderOp::Builder builder;
builder.SetDatasetFilesList({dataset_path}) builder.SetDatasetFilesList({dataset_path}).SetRowsPerBuffer(16);
.SetRowsPerBuffer(16)
.SetNumWorkers(16);
std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>(); std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>();
schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {}); schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {});
builder.SetDataSchema(std::move(schema)); builder.SetDataSchema(std::move(schema));
@ -98,9 +96,7 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderLargeRowsPerBuffer) {
std::shared_ptr<TFReaderOp> my_tfreader_op; std::shared_ptr<TFReaderOp> my_tfreader_op;
TFReaderOp::Builder builder; TFReaderOp::Builder builder;
builder.SetDatasetFilesList({dataset_path}) builder.SetDatasetFilesList({dataset_path}).SetRowsPerBuffer(500);
.SetRowsPerBuffer(500)
.SetNumWorkers(16);
std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>(); std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>();
schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {}); schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {});
builder.SetDataSchema(std::move(schema)); builder.SetDataSchema(std::move(schema));
@ -152,9 +148,7 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderSmallRowsPerBuffer) {
std::shared_ptr<TFReaderOp> my_tfreader_op; std::shared_ptr<TFReaderOp> my_tfreader_op;
TFReaderOp::Builder builder; TFReaderOp::Builder builder;
builder.SetDatasetFilesList({dataset_path}) builder.SetDatasetFilesList({dataset_path}).SetRowsPerBuffer(1);
.SetRowsPerBuffer(1)
.SetNumWorkers(16);
std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>(); std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>();
schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {}); schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {});
builder.SetDataSchema(std::move(schema)); builder.SetDataSchema(std::move(schema));
@ -206,10 +200,7 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderLargeQueueSize) {
std::shared_ptr<TFReaderOp> my_tfreader_op; std::shared_ptr<TFReaderOp> my_tfreader_op;
TFReaderOp::Builder builder; TFReaderOp::Builder builder;
builder.SetDatasetFilesList({dataset_path}) builder.SetDatasetFilesList({dataset_path}).SetWorkerConnectorSize(1).SetRowsPerBuffer(16);
.SetWorkerConnectorSize(1)
.SetRowsPerBuffer(16)
.SetNumWorkers(16);
std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>(); std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>();
schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {}); schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {});
builder.SetDataSchema(std::move(schema)); builder.SetDataSchema(std::move(schema));
@ -316,10 +307,7 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderRepeat) {
// TFReaderOp // TFReaderOp
std::shared_ptr<TFReaderOp> my_tfreader_op; std::shared_ptr<TFReaderOp> my_tfreader_op;
TFReaderOp::Builder builder; TFReaderOp::Builder builder;
builder.SetDatasetFilesList({dataset_path}) builder.SetDatasetFilesList({dataset_path}).SetRowsPerBuffer(16).SetWorkerConnectorSize(16);
.SetRowsPerBuffer(16)
.SetWorkerConnectorSize(16)
.SetNumWorkers(16);
std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>(); std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>();
schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {}); schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {});
builder.SetDataSchema(std::move(schema)); builder.SetDataSchema(std::move(schema));
@ -389,7 +377,7 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderSchemaConstructor) {
std::shared_ptr<TFReaderOp> my_tfreader_op; std::shared_ptr<TFReaderOp> my_tfreader_op;
TFReaderOp::Builder builder; TFReaderOp::Builder builder;
builder.SetDatasetFilesList({dataset_path+"/test.data"}) builder.SetDatasetFilesList({dataset_path + "/test.data"})
.SetRowsPerBuffer(16) .SetRowsPerBuffer(16)
.SetNumWorkers(16) .SetNumWorkers(16)
.SetDataSchema(std::move(data_schema)); .SetDataSchema(std::move(data_schema));
@ -446,7 +434,7 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderTake1Row) {
// TFReaderOp // TFReaderOp
std::shared_ptr<TFReaderOp> my_tfreader_op; std::shared_ptr<TFReaderOp> my_tfreader_op;
TFReaderOp::Builder builder; TFReaderOp::Builder builder;
builder.SetDatasetFilesList({dataset_path + "/test.data"}).SetRowsPerBuffer(5).SetNumWorkers(16); builder.SetDatasetFilesList({dataset_path + "/test.data"}).SetRowsPerBuffer(5);
std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>(); std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>();
schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema1Row.json", {}); schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema1Row.json", {});
builder.SetDataSchema(std::move(schema)); builder.SetDataSchema(std::move(schema));
@ -504,7 +492,7 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderTake1Buffer) {
// TFReaderOp // TFReaderOp
std::shared_ptr<TFReaderOp> my_tfreader_op; std::shared_ptr<TFReaderOp> my_tfreader_op;
TFReaderOp::Builder builder; TFReaderOp::Builder builder;
builder.SetDatasetFilesList({dataset_path + "/test.data"}).SetRowsPerBuffer(5).SetNumWorkers(16); builder.SetDatasetFilesList({dataset_path + "/test.data"}).SetRowsPerBuffer(5);
std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>(); std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>();
schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema5Rows.json", {}); schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema5Rows.json", {});
builder.SetDataSchema(std::move(schema)); builder.SetDataSchema(std::move(schema));
@ -562,7 +550,7 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderTake7Rows) {
// TFReaderOp // TFReaderOp
std::shared_ptr<TFReaderOp> my_tfreader_op; std::shared_ptr<TFReaderOp> my_tfreader_op;
TFReaderOp::Builder builder; TFReaderOp::Builder builder;
builder.SetDatasetFilesList({dataset_path + "/test.data"}).SetRowsPerBuffer(5).SetNumWorkers(16); builder.SetDatasetFilesList({dataset_path + "/test.data"}).SetRowsPerBuffer(5);
std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>(); std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>();
schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema7Rows.json", {}); schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema7Rows.json", {});
builder.SetDataSchema(std::move(schema)); builder.SetDataSchema(std::move(schema));
@ -617,9 +605,7 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderBasicNoSchema) {
std::shared_ptr<TFReaderOp> my_tfreader_op; std::shared_ptr<TFReaderOp> my_tfreader_op;
TFReaderOp::Builder builder; TFReaderOp::Builder builder;
builder.SetDatasetFilesList({dataset_path}) builder.SetDatasetFilesList({dataset_path}).SetRowsPerBuffer(16);
.SetRowsPerBuffer(16)
.SetNumWorkers(16);
Status rc = builder.Build(&my_tfreader_op); Status rc = builder.Build(&my_tfreader_op);
ASSERT_TRUE(rc.IsOk()); ASSERT_TRUE(rc.IsOk());
@ -707,13 +693,11 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderInvalidFiles) {
std::string valid_file = datasets_root_path_ + "/testTFTestAllTypes/test.data"; std::string valid_file = datasets_root_path_ + "/testTFTestAllTypes/test.data";
std::string schema_file = datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json"; std::string schema_file = datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json";
std::string invalid_file = datasets_root_path_ + "/testTFTestAllTypes/invalidFile.txt"; std::string invalid_file = datasets_root_path_ + "/testTFTestAllTypes/invalidFile.txt";
std::string nonexistent_file = "this/file/not/exist"; std::string nonexistent_file = "this/file/does_not/exist";
std::shared_ptr<TFReaderOp> my_tfreader_op; std::shared_ptr<TFReaderOp> my_tfreader_op;
TFReaderOp::Builder builder; TFReaderOp::Builder builder;
builder.SetDatasetFilesList({invalid_file, valid_file, schema_file}) builder.SetDatasetFilesList({invalid_file, valid_file, schema_file}).SetRowsPerBuffer(16);
.SetRowsPerBuffer(16)
.SetNumWorkers(16);
std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>(); std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>();
schema->LoadSchemaFile(schema_file, {}); schema->LoadSchemaFile(schema_file, {});
@ -722,9 +706,7 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderInvalidFiles) {
Status rc = builder.Build(&my_tfreader_op); Status rc = builder.Build(&my_tfreader_op);
ASSERT_TRUE(!rc.IsOk()); ASSERT_TRUE(!rc.IsOk());
builder.SetDatasetFilesList({invalid_file, valid_file, schema_file, nonexistent_file}) builder.SetDatasetFilesList({invalid_file, valid_file, schema_file, nonexistent_file}).SetRowsPerBuffer(16);
.SetRowsPerBuffer(16)
.SetNumWorkers(16);
schema = std::make_unique<DataSchema>(); schema = std::make_unique<DataSchema>();
schema->LoadSchemaFile(schema_file, {}); schema->LoadSchemaFile(schema_file, {});

Write Preview
Loading…
Cancel
Save