From 715d70f397f6e9cc581edc5e4c6c5fe217e11d30 Mon Sep 17 00:00:00 2001 From: hesham Date: Fri, 22 Jan 2021 19:07:48 -0500 Subject: [PATCH] Remove setting num_parallel_worker in UT, since it would fail for machines with less than 16 cores. --- tests/ut/cpp/dataset/c_api_repeat_test.cc | 2 +- tests/ut/cpp/dataset/client_config_test.cc | 2 +- tests/ut/cpp/dataset/clue_op_test.cc | 7 ++- tests/ut/cpp/dataset/concat_op_test.cc | 10 +--- tests/ut/cpp/dataset/csv_op_test.cc | 9 ++-- .../dataset/ir_tensor_op_fusion_pass_test.cc | 2 - tests/ut/cpp/dataset/project_op_test.cc | 5 +- tests/ut/cpp/dataset/skip_op_test.cc | 5 +- tests/ut/cpp/dataset/take_op_test.cc | 5 +- tests/ut/cpp/dataset/text_file_op_test.cc | 10 +--- tests/ut/cpp/dataset/tfReader_op_test.cc | 50 ++++++------------- 11 files changed, 32 insertions(+), 75 deletions(-) diff --git a/tests/ut/cpp/dataset/c_api_repeat_test.cc b/tests/ut/cpp/dataset/c_api_repeat_test.cc index 35df7c46e6..4b302b29ed 100644 --- a/tests/ut/cpp/dataset/c_api_repeat_test.cc +++ b/tests/ut/cpp/dataset/c_api_repeat_test.cc @@ -28,7 +28,7 @@ TEST_F(MindDataTestPipeline, TestRepeatSetNumWorkers) { std::string file_path = datasets_root_path_ + "/testTFTestAllTypes/test.data"; std::shared_ptr ds = TFRecord({file_path}); - ds = ds->SetNumWorkers(16); + ds = ds->SetNumWorkers(8); ds = ds->Repeat(32); // Create an iterator over the result of the above dataset diff --git a/tests/ut/cpp/dataset/client_config_test.cc b/tests/ut/cpp/dataset/client_config_test.cc index 5cc9600b4e..a22d9f6b51 100644 --- a/tests/ut/cpp/dataset/client_config_test.cc +++ b/tests/ut/cpp/dataset/client_config_test.cc @@ -77,7 +77,7 @@ TEST_F(MindDataTestClientConfig, TestClientConfig1) { TEST_F(MindDataTestClientConfig, TestClientConfig2) { std::shared_ptr my_conf = GlobalContext::config_manager(); - my_conf->set_num_parallel_workers(16); + my_conf->set_num_parallel_workers(8); Status rc; diff --git a/tests/ut/cpp/dataset/clue_op_test.cc b/tests/ut/cpp/dataset/clue_op_test.cc index 170e54b7f2..e9be170cd6 100644 --- a/tests/ut/cpp/dataset/clue_op_test.cc +++ b/tests/ut/cpp/dataset/clue_op_test.cc @@ -50,10 +50,9 @@ TEST_F(MindDataTestCLUEOp, TestCLUEBasic) { std::shared_ptr op; ClueOp::Builder builder; builder.SetClueFilesList({dataset_path}) - .SetRowsPerBuffer(16) - .SetNumWorkers(16) - .SetOpConnectorSize(2) - .SetColsKeyMap(key_map); + .SetRowsPerBuffer(16) + .SetOpConnectorSize(2) + .SetColsKeyMap(key_map); Status rc = builder.Build(&op); ASSERT_TRUE(rc.IsOk()); diff --git a/tests/ut/cpp/dataset/concat_op_test.cc b/tests/ut/cpp/dataset/concat_op_test.cc index 7c6d847540..f5915ea2be 100644 --- a/tests/ut/cpp/dataset/concat_op_test.cc +++ b/tests/ut/cpp/dataset/concat_op_test.cc @@ -51,10 +51,7 @@ TEST_F(MindDataTestConcatOp, TestConcatProject) { // TFReaderOp1 std::shared_ptr my_tfreader_op1; TFReaderOp::Builder builder1; - builder1.SetDatasetFilesList({dataset_path}) - .SetRowsPerBuffer(16) - .SetWorkerConnectorSize(16) - .SetNumWorkers(16); + builder1.SetDatasetFilesList({dataset_path}).SetRowsPerBuffer(16).SetWorkerConnectorSize(16); std::unique_ptr schema1 = std::make_unique(); schema1->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {}); builder1.SetDataSchema(std::move(schema1)); @@ -66,10 +63,7 @@ TEST_F(MindDataTestConcatOp, TestConcatProject) { // TFReaderOp2 std::shared_ptr my_tfreader_op2; TFReaderOp::Builder builder2; - builder2.SetDatasetFilesList({dataset_path}) - .SetRowsPerBuffer(16) - .SetWorkerConnectorSize(16) - .SetNumWorkers(16); + builder2.SetDatasetFilesList({dataset_path}).SetRowsPerBuffer(16).SetWorkerConnectorSize(16); std::unique_ptr schema2 = std::make_unique(); schema2->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {}); builder2.SetDataSchema(std::move(schema2)); diff --git a/tests/ut/cpp/dataset/csv_op_test.cc b/tests/ut/cpp/dataset/csv_op_test.cc index 4c01af9654..8e5f8ebddc 100644 --- a/tests/ut/cpp/dataset/csv_op_test.cc +++ b/tests/ut/cpp/dataset/csv_op_test.cc @@ -52,11 +52,10 @@ TEST_F(MindDataTestCSVOp, TestCSVBasic) { std::shared_ptr op; CsvOp::Builder builder; builder.SetCsvFilesList({dataset_path}) - .SetRowsPerBuffer(16) - .SetNumWorkers(16) - .SetShuffleFiles(false) - .SetOpConnectorSize(2) - .SetFieldDelim(',') + .SetRowsPerBuffer(16) + .SetShuffleFiles(false) + .SetOpConnectorSize(2) + .SetFieldDelim(',') .SetColumDefault(column_default_list) .SetColumName({"col1", "col2", "col3", "col4"}); diff --git a/tests/ut/cpp/dataset/ir_tensor_op_fusion_pass_test.cc b/tests/ut/cpp/dataset/ir_tensor_op_fusion_pass_test.cc index b3cc9da7f3..e2465da1b2 100644 --- a/tests/ut/cpp/dataset/ir_tensor_op_fusion_pass_test.cc +++ b/tests/ut/cpp/dataset/ir_tensor_op_fusion_pass_test.cc @@ -37,7 +37,6 @@ TEST_F(MindDataTestTensorOpFusionPass, RandomCropDecodeResizeDisabled) { std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::shared_ptr ds = ImageFolder(folder_path, false, SequentialSampler(0, 11)); - ds = ds->SetNumWorkers(16); // Create objects for the tensor ops std::shared_ptr decode = vision::Decode(); @@ -69,7 +68,6 @@ TEST_F(MindDataTestTensorOpFusionPass, RandomCropDecodeResizeEnabled) { std::string folder_path = datasets_root_path_ + "/testPK/data/"; std::shared_ptr ds = ImageFolder(folder_path, false, SequentialSampler(0, 11)); - ds = ds->SetNumWorkers(16); // Create objects for the tensor ops std::shared_ptr decode = vision::Decode(); diff --git a/tests/ut/cpp/dataset/project_op_test.cc b/tests/ut/cpp/dataset/project_op_test.cc index 5bdb2923d4..1f5d24cf7b 100644 --- a/tests/ut/cpp/dataset/project_op_test.cc +++ b/tests/ut/cpp/dataset/project_op_test.cc @@ -41,10 +41,7 @@ TEST_F(MindDataTestProjectOp, TestProjectProject) { std::shared_ptr my_tfreader_op; TFReaderOp::Builder builder; - builder.SetDatasetFilesList({dataset_path}) - .SetRowsPerBuffer(16) - .SetWorkerConnectorSize(16) - .SetNumWorkers(16); + builder.SetDatasetFilesList({dataset_path}).SetRowsPerBuffer(16).SetWorkerConnectorSize(16); std::unique_ptr schema = std::make_unique(); schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {}); builder.SetDataSchema(std::move(schema)); diff --git a/tests/ut/cpp/dataset/skip_op_test.cc b/tests/ut/cpp/dataset/skip_op_test.cc index 387d2f69ff..e5a9811c52 100644 --- a/tests/ut/cpp/dataset/skip_op_test.cc +++ b/tests/ut/cpp/dataset/skip_op_test.cc @@ -35,10 +35,7 @@ TEST_F(MindDataTestSkipOp, TestSkipOpFuntions) { std::shared_ptr my_tfreader_op; TFReaderOp::Builder builder; - builder.SetDatasetFilesList({dataset_path}) - .SetRowsPerBuffer(16) - .SetWorkerConnectorSize(16) - .SetNumWorkers(16); + builder.SetDatasetFilesList({dataset_path}).SetRowsPerBuffer(16).SetWorkerConnectorSize(16); std::unique_ptr schema = std::make_unique(); schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {}); builder.SetDataSchema(std::move(schema)); diff --git a/tests/ut/cpp/dataset/take_op_test.cc b/tests/ut/cpp/dataset/take_op_test.cc index 156a76e9c1..9b49c39201 100644 --- a/tests/ut/cpp/dataset/take_op_test.cc +++ b/tests/ut/cpp/dataset/take_op_test.cc @@ -42,10 +42,7 @@ TEST_F(MindDataTestTakeOp, TestTakeProject) { // TFReaderOp std::shared_ptr my_tfreader_op; TFReaderOp::Builder builder; - builder.SetDatasetFilesList({dataset_path}) - .SetRowsPerBuffer(16) - .SetWorkerConnectorSize(16) - .SetNumWorkers(16); + builder.SetDatasetFilesList({dataset_path}).SetRowsPerBuffer(16).SetWorkerConnectorSize(16); std::unique_ptr schema = std::make_unique(); schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {}); builder.SetDataSchema(std::move(schema)); diff --git a/tests/ut/cpp/dataset/text_file_op_test.cc b/tests/ut/cpp/dataset/text_file_op_test.cc index 9d99f2b15b..09cd1b770f 100644 --- a/tests/ut/cpp/dataset/text_file_op_test.cc +++ b/tests/ut/cpp/dataset/text_file_op_test.cc @@ -45,10 +45,7 @@ TEST_F(MindDataTestTextFileOp, TestTextFileBasic) { std::shared_ptr op; TextFileOp::Builder builder; - builder.SetTextFilesList({dataset_path}) - .SetRowsPerBuffer(16) - .SetNumWorkers(16) - .SetOpConnectorSize(2); + builder.SetTextFilesList({dataset_path}).SetRowsPerBuffer(16).SetOpConnectorSize(2); Status rc = builder.Build(&op); ASSERT_TRUE(rc.IsOk()); @@ -97,10 +94,7 @@ TEST_F(MindDataTestTextFileOp, TestTextFileFileNotExist) { std::shared_ptr op; TextFileOp::Builder builder; - builder.SetTextFilesList({dataset_path}) - .SetRowsPerBuffer(16) - .SetNumWorkers(16) - .SetOpConnectorSize(2); + builder.SetTextFilesList({dataset_path}).SetRowsPerBuffer(16).SetOpConnectorSize(2); Status rc = builder.Build(&op); ASSERT_TRUE(rc.IsOk()); diff --git a/tests/ut/cpp/dataset/tfReader_op_test.cc b/tests/ut/cpp/dataset/tfReader_op_test.cc index 5577a51e05..4422a81728 100644 --- a/tests/ut/cpp/dataset/tfReader_op_test.cc +++ b/tests/ut/cpp/dataset/tfReader_op_test.cc @@ -44,9 +44,7 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderBasic1) { std::shared_ptr my_tfreader_op; TFReaderOp::Builder builder; - builder.SetDatasetFilesList({dataset_path}) - .SetRowsPerBuffer(16) - .SetNumWorkers(16); + builder.SetDatasetFilesList({dataset_path}).SetRowsPerBuffer(16); std::unique_ptr schema = std::make_unique(); schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {}); builder.SetDataSchema(std::move(schema)); @@ -98,10 +96,8 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderLargeRowsPerBuffer) { std::shared_ptr my_tfreader_op; TFReaderOp::Builder builder; - builder.SetDatasetFilesList({dataset_path}) - .SetRowsPerBuffer(500) - .SetNumWorkers(16); - std::unique_ptr schema = std::make_unique(); + builder.SetDatasetFilesList({dataset_path}).SetRowsPerBuffer(500); + std::unique_ptr schema = std::make_unique(); schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {}); builder.SetDataSchema(std::move(schema)); Status rc = builder.Build(&my_tfreader_op); @@ -152,10 +148,8 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderSmallRowsPerBuffer) { std::shared_ptr my_tfreader_op; TFReaderOp::Builder builder; - builder.SetDatasetFilesList({dataset_path}) - .SetRowsPerBuffer(1) - .SetNumWorkers(16); - std::unique_ptr schema = std::make_unique(); + builder.SetDatasetFilesList({dataset_path}).SetRowsPerBuffer(1); + std::unique_ptr schema = std::make_unique(); schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {}); builder.SetDataSchema(std::move(schema)); Status rc = builder.Build(&my_tfreader_op); @@ -206,11 +200,8 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderLargeQueueSize) { std::shared_ptr my_tfreader_op; TFReaderOp::Builder builder; - builder.SetDatasetFilesList({dataset_path}) - .SetWorkerConnectorSize(1) - .SetRowsPerBuffer(16) - .SetNumWorkers(16); - std::unique_ptr schema = std::make_unique(); + builder.SetDatasetFilesList({dataset_path}).SetWorkerConnectorSize(1).SetRowsPerBuffer(16); + std::unique_ptr schema = std::make_unique(); schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {}); builder.SetDataSchema(std::move(schema)); Status rc = builder.Build(&my_tfreader_op); @@ -316,10 +307,7 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderRepeat) { // TFReaderOp std::shared_ptr my_tfreader_op; TFReaderOp::Builder builder; - builder.SetDatasetFilesList({dataset_path}) - .SetRowsPerBuffer(16) - .SetWorkerConnectorSize(16) - .SetNumWorkers(16); + builder.SetDatasetFilesList({dataset_path}).SetRowsPerBuffer(16).SetWorkerConnectorSize(16); std::unique_ptr schema = std::make_unique(); schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {}); builder.SetDataSchema(std::move(schema)); @@ -389,7 +377,7 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderSchemaConstructor) { std::shared_ptr my_tfreader_op; TFReaderOp::Builder builder; - builder.SetDatasetFilesList({dataset_path+"/test.data"}) + builder.SetDatasetFilesList({dataset_path + "/test.data"}) .SetRowsPerBuffer(16) .SetNumWorkers(16) .SetDataSchema(std::move(data_schema)); @@ -446,7 +434,7 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderTake1Row) { // TFReaderOp std::shared_ptr my_tfreader_op; TFReaderOp::Builder builder; - builder.SetDatasetFilesList({dataset_path + "/test.data"}).SetRowsPerBuffer(5).SetNumWorkers(16); + builder.SetDatasetFilesList({dataset_path + "/test.data"}).SetRowsPerBuffer(5); std::unique_ptr schema = std::make_unique(); schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema1Row.json", {}); builder.SetDataSchema(std::move(schema)); @@ -504,7 +492,7 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderTake1Buffer) { // TFReaderOp std::shared_ptr my_tfreader_op; TFReaderOp::Builder builder; - builder.SetDatasetFilesList({dataset_path + "/test.data"}).SetRowsPerBuffer(5).SetNumWorkers(16); + builder.SetDatasetFilesList({dataset_path + "/test.data"}).SetRowsPerBuffer(5); std::unique_ptr schema = std::make_unique(); schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema5Rows.json", {}); builder.SetDataSchema(std::move(schema)); @@ -562,7 +550,7 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderTake7Rows) { // TFReaderOp std::shared_ptr my_tfreader_op; TFReaderOp::Builder builder; - builder.SetDatasetFilesList({dataset_path + "/test.data"}).SetRowsPerBuffer(5).SetNumWorkers(16); + builder.SetDatasetFilesList({dataset_path + "/test.data"}).SetRowsPerBuffer(5); std::unique_ptr schema = std::make_unique(); schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema7Rows.json", {}); builder.SetDataSchema(std::move(schema)); @@ -617,9 +605,7 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderBasicNoSchema) { std::shared_ptr my_tfreader_op; TFReaderOp::Builder builder; - builder.SetDatasetFilesList({dataset_path}) - .SetRowsPerBuffer(16) - .SetNumWorkers(16); + builder.SetDatasetFilesList({dataset_path}).SetRowsPerBuffer(16); Status rc = builder.Build(&my_tfreader_op); ASSERT_TRUE(rc.IsOk()); @@ -707,13 +693,11 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderInvalidFiles) { std::string valid_file = datasets_root_path_ + "/testTFTestAllTypes/test.data"; std::string schema_file = datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json"; std::string invalid_file = datasets_root_path_ + "/testTFTestAllTypes/invalidFile.txt"; - std::string nonexistent_file = "this/file/not/exist"; + std::string nonexistent_file = "this/file/does_not/exist"; std::shared_ptr my_tfreader_op; TFReaderOp::Builder builder; - builder.SetDatasetFilesList({invalid_file, valid_file, schema_file}) - .SetRowsPerBuffer(16) - .SetNumWorkers(16); + builder.SetDatasetFilesList({invalid_file, valid_file, schema_file}).SetRowsPerBuffer(16); std::unique_ptr schema = std::make_unique(); schema->LoadSchemaFile(schema_file, {}); @@ -722,9 +706,7 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderInvalidFiles) { Status rc = builder.Build(&my_tfreader_op); ASSERT_TRUE(!rc.IsOk()); - builder.SetDatasetFilesList({invalid_file, valid_file, schema_file, nonexistent_file}) - .SetRowsPerBuffer(16) - .SetNumWorkers(16); + builder.SetDatasetFilesList({invalid_file, valid_file, schema_file, nonexistent_file}).SetRowsPerBuffer(16); schema = std::make_unique(); schema->LoadSchemaFile(schema_file, {});