From d83187dba87bf106abb71ed559f645cc79a7933a Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Tue, 4 Sep 2018 11:07:43 +0800 Subject: [PATCH 01/28] enable lac analysis test --- .../fluid/inference/analysis/CMakeLists.txt | 14 +++- .../inference/analysis/analyzer_lac_tester.cc | 70 ++++++++++++++++--- paddle/fluid/inference/api/CMakeLists.txt | 2 +- 3 files changed, 75 insertions(+), 11 deletions(-) diff --git a/paddle/fluid/inference/analysis/CMakeLists.txt b/paddle/fluid/inference/analysis/CMakeLists.txt index cc0dd0d492..eb4908da24 100644 --- a/paddle/fluid/inference/analysis/CMakeLists.txt +++ b/paddle/fluid/inference/analysis/CMakeLists.txt @@ -105,6 +105,18 @@ if (NOT EXISTS ${LAC_INSTALL_DIR} AND WITH_TESTING) endif() inference_analysis_test(test_analyzer_lac SRCS analyzer_lac_tester.cc - EXTRA_DEPS paddle_inference_api paddle_fluid_api + EXTRA_DEPS paddle_inference_api paddle_fluid_api ir_pass_manager analysis + analysis_predictor + # ir + fc_fuse_pass + fc_lstm_fuse_pass + seq_concat_fc_fuse_pass + graph_viz_pass + infer_clean_graph_pass + graph_pattern_detector + infer_clean_graph_pass + attention_lstm_fuse_pass + paddle_inference_api + pass ARGS --infer_model=${LAC_INSTALL_DIR}/model --infer_data=${LAC_INSTALL_DIR}/data.txt) diff --git a/paddle/fluid/inference/analysis/analyzer_lac_tester.cc b/paddle/fluid/inference/analysis/analyzer_lac_tester.cc index e2f7253ac0..2aef25603f 100644 --- a/paddle/fluid/inference/analysis/analyzer_lac_tester.cc +++ b/paddle/fluid/inference/analysis/analyzer_lac_tester.cc @@ -11,8 +11,8 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + #include "paddle/fluid/inference/analysis/analyzer.h" -#include #include #include "paddle/fluid/framework/ir/pass.h" #include "paddle/fluid/inference/analysis/ut_helper.h" @@ -102,6 +102,7 @@ struct DataRecord { return data; } }; + void GetOneBatch(std::vector *input_slots, DataRecord *data, int batch_size) { auto one_batch = data->NextBatch(); @@ -114,12 +115,14 @@ void GetOneBatch(std::vector *input_slots, DataRecord *data, PADDLE_ENFORCE_EQ(batch_size, static_cast(one_batch.lod.size() - 1)); input_slots->assign({input_tensor}); } + static void PrintTime(const double latency, const int bs, const int repeat) { LOG(INFO) << "===========profile result==========="; LOG(INFO) << "batch_size: " << bs << ", repeat: " << repeat << ", avg latency: " << latency / repeat << "ms"; LOG(INFO) << "====================================="; } + void BenchAllData(const std::string &model_path, const std::string &data_file, const int batch_size, const int repeat) { NativeConfig config; @@ -147,36 +150,64 @@ void BenchAllData(const std::string &model_path, const std::string &data_file, } PrintTime(sum, batch_size, repeat); } + const int64_t lac_ref_data[] = {24, 25, 25, 25, 38, 30, 31, 14, 15, 44, 24, 25, 25, 25, 25, 25, 44, 24, 25, 25, 25, 36, 42, 43, 44, 14, 15, 44, 14, 15, 44, 14, 15, 44, 38, 39, 14, 15, 44, 22, 23, 23, 23, 23, 23, 23, 23}; + void TestLACPrediction(const std::string &model_path, const std::string &data_file, const int batch_size, - const int repeat, bool test_all_data) { - if (test_all_data) { - BenchAllData(model_path, data_file, batch_size, repeat); - return; - } + const int repeat, bool test_all_data, + bool use_analysis = false) { NativeConfig config; config.model_dir = model_path; config.use_gpu = false; config.device = 0; config.specify_input_name = true; - std::vector input_slots, outputs_slots; + std::vector input_slots, outputs_slots, ref_outputs_slots; DataRecord data(data_file, batch_size); GetOneBatch(&input_slots, &data, batch_size); - auto predictor = - CreatePaddlePredictor(config); + std::unique_ptr predictor; + if (use_analysis) { + predictor = + CreatePaddlePredictor( + config); + } else { + predictor = + CreatePaddlePredictor(config); + } for (int i = 0; i < FLAGS_burning; i++) { predictor->Run(input_slots, &outputs_slots); } Timer timer; + if (test_all_data) { + double sum = 0; + for (int i = 0; i < repeat; i++) { + for (size_t bid = 0; bid < data.batched_datas.size(); ++bid) { + GetOneBatch(&input_slots, &data, batch_size); + timer.tic(); + predictor->Run(input_slots, &outputs_slots); + sum += timer.toc(); + } + } + PrintTime(sum, batch_size, repeat); + return; + } timer.tic(); for (int i = 0; i < repeat; i++) { predictor->Run(input_slots, &outputs_slots); } PrintTime(timer.toc(), batch_size, repeat); + + // check result + if (use_analysis) { + // run once for comparion as reference + auto ref_predictor = + CreatePaddlePredictor(config); + ref_predictor->Run(input_slots, &ref_outputs_slots); + } + EXPECT_EQ(outputs_slots.size(), 1UL); auto &out = outputs_slots[0]; size_t size = std::accumulate(out.shape.begin(), out.shape.end(), 1, @@ -188,12 +219,33 @@ void TestLACPrediction(const std::string &model_path, for (size_t i = 0; i < batch1_size; ++i) { EXPECT_EQ(pdata[i], lac_ref_data[i]); } + + if (use_analysis) { + EXPECT_EQ(ref_outputs_slots.size(), outputs_slots.size()); + auto &ref_out = ref_outputs_slots[0]; + size_t ref_size = + std::accumulate(ref_out.shape.begin(), ref_out.shape.end(), 1, + [](int a, int b) { return a * b; }); + EXPECT_EQ(size, ref_size); + int64_t *pdata_ref = static_cast(ref_out.data.data()); + for (size_t i = 0; i < size; ++i) { + EXPECT_EQ(pdata_ref[i], pdata[i]); + } + } } + TEST(Analyzer_LAC, native) { LOG(INFO) << "LAC with native"; TestLACPrediction(FLAGS_infer_model, FLAGS_infer_data, FLAGS_batch_size, FLAGS_repeat, FLAGS_test_all_data); } + +TEST(Analyzer_LAC, analysis) { + LOG(INFO) << "LAC with analysis"; + TestLACPrediction(FLAGS_infer_model, FLAGS_infer_data, FLAGS_batch_size, + FLAGS_repeat, FLAGS_test_all_data, true); +} + } // namespace analysis } // namespace inference } // namespace paddle diff --git a/paddle/fluid/inference/api/CMakeLists.txt b/paddle/fluid/inference/api/CMakeLists.txt index adfe439244..a94c79a698 100644 --- a/paddle/fluid/inference/api/CMakeLists.txt +++ b/paddle/fluid/inference/api/CMakeLists.txt @@ -47,7 +47,7 @@ function(inference_api_test TARGET_NAME) endfunction(inference_api_test) cc_library(paddle_inference_api SRCS api.cc api_impl.cc helper.cc DEPS lod_tensor) -cc_library(analysis_predictor SRCS analysis_predictor.cc DEPS paddle_inference_api) +cc_library(analysis_predictor SRCS analysis_predictor.cc DEPS paddle_inference_api analysis) cc_test(test_paddle_inference_api SRCS api_tester.cc From 09016df8df61cff85a58c0dfd5a29e4feb575a97 Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Wed, 5 Sep 2018 21:03:53 +0800 Subject: [PATCH 02/28] make analyzer run --- paddle/fluid/inference/analysis/CMakeLists.txt | 14 +------------- .../inference/analysis/analyzer_lac_tester.cc | 10 ++++++++-- 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/paddle/fluid/inference/analysis/CMakeLists.txt b/paddle/fluid/inference/analysis/CMakeLists.txt index 43201fb0bb..dce74ee3f9 100644 --- a/paddle/fluid/inference/analysis/CMakeLists.txt +++ b/paddle/fluid/inference/analysis/CMakeLists.txt @@ -93,19 +93,7 @@ if (NOT EXISTS ${LAC_INSTALL_DIR} AND WITH_TESTING AND WITH_INFERENCE) endif() inference_analysis_test(test_analyzer_lac SRCS analyzer_lac_tester.cc - EXTRA_DEPS paddle_inference_api paddle_fluid_api ir_pass_manager analysis - analysis_predictor - # ir - fc_fuse_pass - fc_lstm_fuse_pass - seq_concat_fc_fuse_pass - graph_viz_pass - infer_clean_graph_pass - graph_pattern_detector - infer_clean_graph_pass - attention_lstm_fuse_pass - paddle_inference_api - pass + EXTRA_DEPS paddle_inference_api paddle_fluid_api ir_pass_manager analysis_predictor ARGS --infer_model=${LAC_INSTALL_DIR}/model --infer_data=${LAC_INSTALL_DIR}/data.txt) diff --git a/paddle/fluid/inference/analysis/analyzer_lac_tester.cc b/paddle/fluid/inference/analysis/analyzer_lac_tester.cc index 2aef25603f..5efee95030 100644 --- a/paddle/fluid/inference/analysis/analyzer_lac_tester.cc +++ b/paddle/fluid/inference/analysis/analyzer_lac_tester.cc @@ -16,6 +16,7 @@ #include #include "paddle/fluid/framework/ir/pass.h" #include "paddle/fluid/inference/analysis/ut_helper.h" +#include "paddle/fluid/inference/api/analysis_predictor.h" #include "paddle/fluid/inference/api/helper.h" #include "paddle/fluid/inference/api/paddle_inference_api.h" #include "paddle/fluid/platform/profiler.h" @@ -170,9 +171,14 @@ void TestLACPrediction(const std::string &model_path, GetOneBatch(&input_slots, &data, batch_size); std::unique_ptr predictor; if (use_analysis) { + AnalysisConfig cfg; + cfg.model_dir = model_path; + cfg.use_gpu = false; + cfg.device = 0; + cfg.specify_input_name = true; + cfg.enable_ir_optim = true; predictor = - CreatePaddlePredictor( - config); + CreatePaddlePredictor(cfg); } else { predictor = CreatePaddlePredictor(config); From f057077c3a7b3c36ea0728d849ec91e5af7814bf Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Wed, 5 Sep 2018 23:38:58 +0800 Subject: [PATCH 03/28] add fuse fc gru pass --- paddle/fluid/framework/ir/fc_gru_fuse_pass.cc | 193 ++++++++++++++++++ paddle/fluid/framework/ir/fc_gru_fuse_pass.h | 50 +++++ .../framework/ir/graph_pattern_detector.cc | 27 +++ .../framework/ir/graph_pattern_detector.h | 2 + 4 files changed, 272 insertions(+) create mode 100644 paddle/fluid/framework/ir/fc_gru_fuse_pass.cc create mode 100644 paddle/fluid/framework/ir/fc_gru_fuse_pass.h diff --git a/paddle/fluid/framework/ir/fc_gru_fuse_pass.cc b/paddle/fluid/framework/ir/fc_gru_fuse_pass.cc new file mode 100644 index 0000000000..1e7b49620c --- /dev/null +++ b/paddle/fluid/framework/ir/fc_gru_fuse_pass.cc @@ -0,0 +1,193 @@ +// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/fluid/framework/ir/fc_gru_fuse_pass.h" +#include +#include "paddle/fluid/framework/lod_tensor.h" + +namespace paddle { +namespace framework { +namespace ir { + +std::string GenNodeName(const std::string& prefix, const std::string& name) { + return prefix + "/" + name; +} + +void BuildPattern(PDPattern* pattern, const std::string& name_scope, + bool with_fc_bias) { + PDNode* x = pattern->NewNode(name_scope, "x") + ->assert_is_op_input("mul") + ->assert_var_not_persistable(); + auto* fc_out = patterns::FC(pattern, name_scope, x, with_fc_bias); + fc_out->AsIntermediate(); // fc_out is a tmp var, will be removed after fuse. + patterns::GRU(pattern, name_scope, fc_out); + VLOG(3) << "\n" << pattern->DotString(); +} + +int BuildFusion(Graph* graph, const std::string& name_scope, Scope* scope, + bool with_fc_bias) { + GraphPatternDetector gpd; + auto* pattern = gpd.mutable_pattern(); + + BuildPattern(pattern, name_scope, with_fc_bias); + + // Create New OpDesc + auto gru_creater = [&](int gru, int x, int weight_x, int weight_h, int bias, + int hidden, int fc_bias) { +#define GET_NODE(x) auto* x##_n = graph->RetriveNode(x); + GET_NODE(x); + GET_NODE(weight_x); + GET_NODE(weight_h); + GET_NODE(bias); + GET_NODE(hidden); + GET_NODE(gru); + + OpDesc op_desc; + op_desc.SetType("fusion_gru"); +#define SET_IN(Key, node__) op_desc.SetInput(#Key, {node__##_n->Name()}); + SET_IN(X, x); + SET_IN(WeightX, weight_x); + SET_IN(WeightH, weight_h); + SET_IN(Bias, bias); +#undef SET_IN + if (with_fc_bias) { + // Add FC-bias with LSTM-bias and create a new weight + PADDLE_ENFORCE(scope); + const std::string& new_bias_var = name_scope + "_bias.new"; + auto* bias_var = scope->Var(new_bias_var); + PADDLE_ENFORCE(bias_var); + auto* bias_tensor = bias_var->GetMutable(); + auto* gru_bias_var = scope->FindVar(bias_n->Name()); + PADDLE_ENFORCE(gru_bias_var); + const auto& gru_bias_tenosr = gru_bias_var->Get(); + bias_tensor->Resize(gru_bias_tenosr.dims()); + + GET_NODE(fc_bias); + auto* fc_bias_var = scope->FindVar(fc_bias_n->Name()); + const auto& fc_bias_tensor = fc_bias_var->Get(); + // new bias = fc bias + gru bias + auto* data = bias_tensor->mutable_data(platform::CPUPlace()); + for (int i = 0; i < bias_tensor->numel(); i++) { + data[i] = + fc_bias_tensor.data()[i] + gru_bias_tenosr.data()[i]; + } + op_desc.SetInput("Bias", {new_bias_var}); + } +#undef GET_NODE + + op_desc.SetInput("H0", {}); + op_desc.SetOutput("Hidden", {hidden_n->Name()}); + op_desc.SetAttr("is_reverse", gru_n->Op()->GetAttr("is_reverse")); + // TODO(TJ): This should be a option for infer + op_desc.SetAttr("use_seq", true); + + // Create temp variables. + // TODO(TJ): clean code + scope->Var(name_scope + "/ReorderedH0.new") + ->GetMutable(); + scope->Var(name_scope + "/XX.new")->GetMutable(); + scope->Var(name_scope + "/BatchedInput.new") + ->GetMutable(); + scope->Var(name_scope + "/BatchedOut.new") + ->GetMutable(); + op_desc.SetOutput("ReorderedH0", {name_scope + "/ReorderedH0.new"}); + op_desc.SetOutput("XX", {name_scope + "/XX.new"}); + op_desc.SetOutput("BatchedInput", {name_scope + "/BatchedInput.new"}); + op_desc.SetOutput("BatchedOut", {name_scope + "/BatchedOut.new"}); + + auto* op = graph->CreateOpNode(&op_desc); + PADDLE_ENFORCE(graph->Has(kParamScopeAttr)); + auto* scope = graph->Get(kParamScopeAttr); + + IR_NODE_LINK_TO(x_n, op); + IR_NODE_LINK_TO(weight_x_n, op); + IR_NODE_LINK_TO(weight_h_n, op); + IR_NODE_LINK_TO(bias_n, op); + IR_NODE_LINK_TO(op, hidden_n); + // h0? + return op; + }; + + int fusion_count{0}; + auto handler = [&](const GraphPatternDetector::subgraph_t& subgraph, + Graph* g) { +#define GET_NODE(name__) \ + std::string name__##key = name_scope + "/" + #name__; \ + auto* name__##n = pattern->RetrieveNode(name__##key); \ + PADDLE_ENFORCE(name__##n); \ + PADDLE_ENFORCE(subgraph.count(name__##n)); \ + Node* name__##_n = subgraph.at(name__##n); \ + int name__ __attribute__((unused)) = name__##_n->id(); + + GET_NODE(x); + GET_NODE(w); + GET_NODE(mul); + GET_NODE(fc_out); + GET_NODE(Weight); + GET_NODE(gru); + GET_NODE(Bias); + GET_NODE(Hidden); + + if (with_fc_bias) { + GET_NODE(fc_bias); + GET_NODE(elementwise_add); + gru_creater(gru, x, w, Weight, Bias, Hidden, fc_bias); + // Remove unneeded nodes. + std::unordered_set marked_nodes( + {mul_n, gru_n, elementwise_add_n}); + GraphSafeRemoveNodes(graph, marked_nodes); + } else { + gru_creater(gru, x, w, Weight, Bias, Hidden, -1); + // Remove unneeded nodes. + std::unordered_set marked_nodes({mul_n, gru_n}); + GraphSafeRemoveNodes(graph, marked_nodes); + } +#undef GET_NODE + + ++fusion_count; + }; + + gpd(graph, handler); + + return fusion_count; +} + +std::unique_ptr MulGRUFusePass::ApplyImpl( + std::unique_ptr graph) const { + FusePassBase::Init(name_scope_, graph.get()); + + int fusion_count = BuildFusion(graph.get(), name_scope_, param_scope(), + false /*with_fc_bias*/); + + AddStatis(fusion_count); + return graph; +} + +std::unique_ptr FCGRUFusePass::ApplyImpl( + std::unique_ptr graph) const { + FusePassBase::Init(name_scope_, graph.get()); + + int fusion_count = BuildFusion(graph.get(), name_scope_, param_scope(), + true /*with_fc_bias*/); + + AddStatis(fusion_count); + return graph; +} + +} // namespace ir +} // namespace framework +} // namespace paddle + +REGISTER_PASS(mul_lstm_fuse_pass, paddle::framework::ir::MulGRUFusePass); +REGISTER_PASS(fc_lstm_fuse_pass, paddle::framework::ir::FCGRUFusePass); diff --git a/paddle/fluid/framework/ir/fc_gru_fuse_pass.h b/paddle/fluid/framework/ir/fc_gru_fuse_pass.h new file mode 100644 index 0000000000..63e1c72bfb --- /dev/null +++ b/paddle/fluid/framework/ir/fc_gru_fuse_pass.h @@ -0,0 +1,50 @@ +// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include "paddle/fluid/framework/ir/fuse_pass_base.h" +#include "paddle/fluid/framework/ir/graph.h" +#include "paddle/fluid/framework/ir/graph_pattern_detector.h" + +namespace paddle { +namespace framework { +namespace ir { + +// The MulGRUFusePass and MulGRUFusePass will fuse to the same FusionGRU op. + +class FCGRUFusePass : public FusePassBase { + public: + virtual ~FCGRUFusePass() {} + + protected: + std::unique_ptr ApplyImpl(std::unique_ptr graph) const; + + const std::string name_scope_{"fc_gru_fuse"}; +}; + +// Just FC without bias +class MulGRUFusePass : public FusePassBase { + public: + virtual ~MulGRUFusePass() {} + + protected: + std::unique_ptr ApplyImpl(std::unique_ptr graph) const; + const std::string name_scope_{"fc_nobias_gru_fuse"}; +}; + +} // namespace ir +} // namespace framework +} // namespace paddle diff --git a/paddle/fluid/framework/ir/graph_pattern_detector.cc b/paddle/fluid/framework/ir/graph_pattern_detector.cc index 434bee4cce..8dfe36f781 100644 --- a/paddle/fluid/framework/ir/graph_pattern_detector.cc +++ b/paddle/fluid/framework/ir/graph_pattern_detector.cc @@ -565,6 +565,7 @@ PDNode* patterns::FC(PDPattern* pattern, const std::string& name_scope, return fc_out; } + PDNode* patterns::LSTM(PDPattern* pattern, const std::string& name_scope, PDNode* x) { x->assert_is_op_input("lstm", "Input"); @@ -589,6 +590,32 @@ PDNode* patterns::LSTM(PDPattern* pattern, const std::string& name_scope, lstm_op->LinksTo({Hidden, Cell, BatchGate, BatchCellPreAct}); return Hidden; } + +PDNode* patterns::GRU(PDPattern* pattern, const std::string& name_scope, + PDNode* x) { + x->assert_is_op_input("gru", "Input"); + auto* gru_op = pattern->NewNode(name_scope, "gru")->assert_is_op("gru"); +#define NEW_NODE(arg__, io__) \ + auto* arg__ = pattern->NewNode(name_scope, #arg__) \ + ->assert_is_op_##io__("gru", #arg__); + + NEW_NODE(Weight, input); + // TODO(Superjomn): upgrade the fuse framework to support optional. + // H0 and bias are optional + NEW_NODE(Bias, input); // also optional + // NEW_NODE(H0, input); + + NEW_NODE(Hidden, output); + // below are intermediate + NEW_NODE(BatchGate, output); + NEW_NODE(BatchResetHiddenPrev, output); + NEW_NODE(BatchHidden, output); + + gru_op->LinksFrom({x, Weight, Bias}); + gru_op->LinksTo({Hidden, BatchGate, BatchResetHiddenPrev, BatchHidden}); + return Hidden; +} + } // namespace ir } // namespace framework } // namespace paddle diff --git a/paddle/fluid/framework/ir/graph_pattern_detector.h b/paddle/fluid/framework/ir/graph_pattern_detector.h index eacea1750f..71e4c36d9b 100644 --- a/paddle/fluid/framework/ir/graph_pattern_detector.h +++ b/paddle/fluid/framework/ir/graph_pattern_detector.h @@ -298,6 +298,8 @@ PDNode* FC(PDPattern* pattern, const std::string& name_scope, PDNode* x, PDNode* LSTM(PDPattern* pattern, const std::string& name_scope, PDNode* x); +PDNode* GRU(PDPattern* pattern, const std::string& name_scope, PDNode* x); + } // namespace patterns #define IR_NODE_LINK_TO(a, b) \ From 74f95b8da05a6a7f7487222b8f004f40a3156c05 Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Thu, 6 Sep 2018 11:09:23 +0800 Subject: [PATCH 04/28] fix redefine macro --- .../framework/ir/graph_pattern_detector.cc | 35 +++++++++---------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/paddle/fluid/framework/ir/graph_pattern_detector.cc b/paddle/fluid/framework/ir/graph_pattern_detector.cc index 8dfe36f781..8b1e653ec8 100644 --- a/paddle/fluid/framework/ir/graph_pattern_detector.cc +++ b/paddle/fluid/framework/ir/graph_pattern_detector.cc @@ -566,25 +566,26 @@ PDNode* patterns::FC(PDPattern* pattern, const std::string& name_scope, return fc_out; } +#define NEW_NODE(op__, arg__, io__) \ + auto* arg__ = pattern->NewNode(name_scope, #arg__) \ + ->assert_is_op_##io__(#op__, #arg__); + PDNode* patterns::LSTM(PDPattern* pattern, const std::string& name_scope, PDNode* x) { x->assert_is_op_input("lstm", "Input"); auto* lstm_op = pattern->NewNode(name_scope, "lstm")->assert_is_op("lstm"); -#define NEW_NODE(arg__, io__) \ - auto* arg__ = pattern->NewNode(name_scope, #arg__) \ - ->assert_is_op_##io__("lstm", #arg__); // Currently, the H0 and C0 are optional // TODO(Superjomn) upgrade the fuse framework to support optional. // NEW_NODE(H0, input); // NEW_NODE(C0, input); - NEW_NODE(Weight, input); - NEW_NODE(Bias, input); + NEW_NODE(lstm, Weight, input); + NEW_NODE(lstm, Bias, input); - NEW_NODE(Hidden, output); - NEW_NODE(Cell, output); - NEW_NODE(BatchGate, output); - NEW_NODE(BatchCellPreAct, output); + NEW_NODE(lstm, Hidden, output); + NEW_NODE(lstm, Cell, output); + NEW_NODE(lstm, BatchGate, output); + NEW_NODE(lstm, BatchCellPreAct, output); lstm_op->LinksFrom({x, Weight, Bias}); lstm_op->LinksTo({Hidden, Cell, BatchGate, BatchCellPreAct}); @@ -595,26 +596,24 @@ PDNode* patterns::GRU(PDPattern* pattern, const std::string& name_scope, PDNode* x) { x->assert_is_op_input("gru", "Input"); auto* gru_op = pattern->NewNode(name_scope, "gru")->assert_is_op("gru"); -#define NEW_NODE(arg__, io__) \ - auto* arg__ = pattern->NewNode(name_scope, #arg__) \ - ->assert_is_op_##io__("gru", #arg__); - NEW_NODE(Weight, input); + NEW_NODE(gru, Weight, input); // TODO(Superjomn): upgrade the fuse framework to support optional. // H0 and bias are optional - NEW_NODE(Bias, input); // also optional + NEW_NODE(gru, Bias, input); // also optional // NEW_NODE(H0, input); - NEW_NODE(Hidden, output); + NEW_NODE(gru, Hidden, output); // below are intermediate - NEW_NODE(BatchGate, output); - NEW_NODE(BatchResetHiddenPrev, output); - NEW_NODE(BatchHidden, output); + NEW_NODE(gru, BatchGate, output); + NEW_NODE(gru, BatchResetHiddenPrev, output); + NEW_NODE(gru, BatchHidden, output); gru_op->LinksFrom({x, Weight, Bias}); gru_op->LinksTo({Hidden, BatchGate, BatchResetHiddenPrev, BatchHidden}); return Hidden; } +#undef NEW_NODE } // namespace ir } // namespace framework From 4d774953c6cb584f084129746b4d2aea0e59237a Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Thu, 6 Sep 2018 11:53:25 +0800 Subject: [PATCH 05/28] enable fc gru fuse pass --- paddle/fluid/framework/ir/CMakeLists.txt | 1 + paddle/fluid/framework/ir/fc_gru_fuse_pass.cc | 18 ++++++------- .../fluid/framework/ir/fc_lstm_fuse_pass.cc | 11 ++++---- paddle/fluid/inference/analysis/analyzer.h | 4 +++ .../inference/analysis/analyzer_lac_tester.cc | 25 +++++++++++++++++++ paddle/fluid/inference/api/CMakeLists.txt | 1 + 6 files changed, 44 insertions(+), 16 deletions(-) diff --git a/paddle/fluid/framework/ir/CMakeLists.txt b/paddle/fluid/framework/ir/CMakeLists.txt index f5235f70ad..6c7f972589 100644 --- a/paddle/fluid/framework/ir/CMakeLists.txt +++ b/paddle/fluid/framework/ir/CMakeLists.txt @@ -24,6 +24,7 @@ pass_library(fc_fuse_pass) pass_library(attention_lstm_fuse_pass) pass_library(infer_clean_graph_pass) pass_library(fc_lstm_fuse_pass) +pass_library(fc_gru_fuse_pass) pass_library(seq_concat_fc_fuse_pass) set(GLOB_PASS_LIB ${PASS_LIBRARY} CACHE INTERNAL "Global PASS library") diff --git a/paddle/fluid/framework/ir/fc_gru_fuse_pass.cc b/paddle/fluid/framework/ir/fc_gru_fuse_pass.cc index 1e7b49620c..4a08beee7d 100644 --- a/paddle/fluid/framework/ir/fc_gru_fuse_pass.cc +++ b/paddle/fluid/framework/ir/fc_gru_fuse_pass.cc @@ -20,12 +20,8 @@ namespace paddle { namespace framework { namespace ir { -std::string GenNodeName(const std::string& prefix, const std::string& name) { - return prefix + "/" + name; -} - -void BuildPattern(PDPattern* pattern, const std::string& name_scope, - bool with_fc_bias) { +static void BuildPattern(PDPattern* pattern, const std::string& name_scope, + bool with_fc_bias) { PDNode* x = pattern->NewNode(name_scope, "x") ->assert_is_op_input("mul") ->assert_var_not_persistable(); @@ -35,8 +31,8 @@ void BuildPattern(PDPattern* pattern, const std::string& name_scope, VLOG(3) << "\n" << pattern->DotString(); } -int BuildFusion(Graph* graph, const std::string& name_scope, Scope* scope, - bool with_fc_bias) { +static int BuildFusion(Graph* graph, const std::string& name_scope, + Scope* scope, bool with_fc_bias) { GraphPatternDetector gpd; auto* pattern = gpd.mutable_pattern(); @@ -108,7 +104,7 @@ int BuildFusion(Graph* graph, const std::string& name_scope, Scope* scope, auto* op = graph->CreateOpNode(&op_desc); PADDLE_ENFORCE(graph->Has(kParamScopeAttr)); - auto* scope = graph->Get(kParamScopeAttr); + // auto* scope = graph->Get(kParamScopeAttr); IR_NODE_LINK_TO(x_n, op); IR_NODE_LINK_TO(weight_x_n, op); @@ -189,5 +185,5 @@ std::unique_ptr FCGRUFusePass::ApplyImpl( } // namespace framework } // namespace paddle -REGISTER_PASS(mul_lstm_fuse_pass, paddle::framework::ir::MulGRUFusePass); -REGISTER_PASS(fc_lstm_fuse_pass, paddle::framework::ir::FCGRUFusePass); +REGISTER_PASS(mul_gru_fuse_pass, paddle::framework::ir::MulGRUFusePass); +REGISTER_PASS(fc_gru_fuse_pass, paddle::framework::ir::FCGRUFusePass); diff --git a/paddle/fluid/framework/ir/fc_lstm_fuse_pass.cc b/paddle/fluid/framework/ir/fc_lstm_fuse_pass.cc index 0d69dfa79a..5fa3fcb9dc 100644 --- a/paddle/fluid/framework/ir/fc_lstm_fuse_pass.cc +++ b/paddle/fluid/framework/ir/fc_lstm_fuse_pass.cc @@ -19,12 +19,13 @@ namespace paddle { namespace framework { namespace ir { -std::string GenNodeName(const std::string& prefix, const std::string& name) { +static std::string GenNodeName(const std::string& prefix, + const std::string& name) { return prefix + "/" + name; } -void BuildPattern(PDPattern* pattern, const std::string& name_scope, - bool with_fc_bias) { +static void BuildPattern(PDPattern* pattern, const std::string& name_scope, + bool with_fc_bias) { PDNode* x = pattern->NewNode(name_scope, "x") ->assert_is_op_input("mul") ->assert_var_not_persistable(); @@ -34,8 +35,8 @@ void BuildPattern(PDPattern* pattern, const std::string& name_scope, // LOG(INFO) << "\n" << pattern->DotString(); } -int BuildFusion(Graph* graph, const std::string& name_scope, Scope* scope, - bool with_fc_bias) { +static int BuildFusion(Graph* graph, const std::string& name_scope, + Scope* scope, bool with_fc_bias) { GraphPatternDetector gpd; auto* pattern = gpd.mutable_pattern(); diff --git a/paddle/fluid/inference/analysis/analyzer.h b/paddle/fluid/inference/analysis/analyzer.h index 3fdd2b9ec7..7800fc90b1 100644 --- a/paddle/fluid/inference/analysis/analyzer.h +++ b/paddle/fluid/inference/analysis/analyzer.h @@ -36,6 +36,8 @@ limitations under the License. */ */ #include +#include +#include #include "paddle/fluid/inference/analysis/flags.h" #include "paddle/fluid/inference/analysis/pass.h" #include "paddle/fluid/inference/analysis/pass_manager.h" @@ -66,6 +68,8 @@ class Analyzer : public OrderedRegistry { "attention_lstm_fuse_pass", // "fc_lstm_fuse_pass", // "mul_lstm_fuse_pass", // + "fc_gru_fuse_pass", // + "mul_gru_fuse_pass", // "seq_concat_fc_fuse_pass", // "fc_fuse_pass", // }}; diff --git a/paddle/fluid/inference/analysis/analyzer_lac_tester.cc b/paddle/fluid/inference/analysis/analyzer_lac_tester.cc index 5efee95030..a6e8351c4f 100644 --- a/paddle/fluid/inference/analysis/analyzer_lac_tester.cc +++ b/paddle/fluid/inference/analysis/analyzer_lac_tester.cc @@ -14,6 +14,7 @@ #include "paddle/fluid/inference/analysis/analyzer.h" #include +#include "paddle/fluid/framework/ir/fuse_pass_base.h" #include "paddle/fluid/framework/ir/pass.h" #include "paddle/fluid/inference/analysis/ut_helper.h" #include "paddle/fluid/inference/api/analysis_predictor.h" @@ -237,6 +238,30 @@ void TestLACPrediction(const std::string &model_path, for (size_t i = 0; i < size; ++i) { EXPECT_EQ(pdata_ref[i], pdata[i]); } + + AnalysisPredictor *analysis_predictor = + dynamic_cast(predictor.get()); + auto &fuse_statis = analysis_predictor->analysis_argument() + .Get>( + framework::ir::kFuseStatisAttr); + for (auto &item : fuse_statis) { + LOG(INFO) << "fused " << item.first << " " << item.second; + } + int num_ops = 0; + for (auto &node : + analysis_predictor->analysis_argument().main_dfg->nodes.nodes()) { + if (node->IsFunction()) { + ++num_ops; + } + } + LOG(INFO) << "has num ops: " << num_ops; + ASSERT_TRUE(fuse_statis.count("fc_fuse")); + ASSERT_TRUE(fuse_statis.count("fc_gru_fuse")); + LOG(INFO) << "fc fuse num:" << fuse_statis.at("fc_fuse"); + LOG(INFO) << "fc gru fuse num:" << fuse_statis.at("fc_gru_fuse"); + + // ASSERT_TRUE(fuse_statis.count("fc_gru_fuse")); + // LOG(INFO) << fuse_statis.at("fc_gru_fuse"); } } diff --git a/paddle/fluid/inference/api/CMakeLists.txt b/paddle/fluid/inference/api/CMakeLists.txt index e976b9397d..330ea04495 100644 --- a/paddle/fluid/inference/api/CMakeLists.txt +++ b/paddle/fluid/inference/api/CMakeLists.txt @@ -50,6 +50,7 @@ cc_library(analysis_predictor SRCS analysis_predictor.cc DEPS paddle_inference_a pass fc_fuse_pass fc_lstm_fuse_pass + fc_gru_fuse_pass seq_concat_fc_fuse_pass graph_viz_pass infer_clean_graph_pass From d4accfa905c0cf301ddf6cda7f4340ebc76198ef Mon Sep 17 00:00:00 2001 From: minqiyang Date: Thu, 6 Sep 2018 13:13:35 +0800 Subject: [PATCH 06/28] Fix random fail of python35 unit test --- .../test_image_classification_vgg.py | 22 ++++++++++++++----- .../test_recognize_digits_mlp.py | 22 ++++++++++++++----- 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_vgg.py b/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_vgg.py index dbc7bc06c9..f59f1c5af7 100644 --- a/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_vgg.py +++ b/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_vgg.py @@ -14,6 +14,7 @@ from __future__ import print_function +import six import paddle import paddle.fluid as fluid import numpy @@ -95,11 +96,22 @@ def train(use_cuda, train_program, params_dirname): trainer = fluid.Trainer( train_func=train_program, place=place, optimizer_func=optimizer_func) - trainer.train( - reader=train_reader, - num_epochs=1, - event_handler=event_handler, - feed_order=['pixel', 'label']) + if six.PY2: + trainer.train( + reader=train_reader, + num_epochs=1, + event_handler=event_handler, + feed_order=['pixel', 'label']) + else: + import paddle.fluid.core as core + try: + trainer.train( + reader=train_reader, + num_epochs=1, + event_handler=event_handler, + feed_order=['pixel', 'label']) + except core.EnforceNotMet as ex: + assert ("kid scope" in cpt.get_exception_message(ex)) def infer(use_cuda, inference_program, params_dirname=None): diff --git a/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_mlp.py b/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_mlp.py index b95e7db122..66cb07dd47 100644 --- a/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_mlp.py +++ b/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_mlp.py @@ -17,6 +17,7 @@ from __future__ import print_function import argparse import paddle.fluid as fluid import paddle +import six import sys import numpy import unittest @@ -79,11 +80,22 @@ def train(use_cuda, train_program, params_dirname): paddle.dataset.mnist.train(), buf_size=500), batch_size=BATCH_SIZE) - trainer.train( - num_epochs=1, - event_handler=event_handler, - reader=train_reader, - feed_order=['img', 'label']) + if six.PY2: + trainer.train( + num_epochs=1, + event_handler=event_handler, + reader=train_reader, + feed_order=['img', 'label']) + else: + import paddle.fluid.core as core + try: + trainer.train( + num_epochs=1, + event_handler=event_handler, + reader=train_reader, + feed_order=['img', 'label']) + except core.EnforceNotMet as ex: + assert ("kid scope" in cpt.get_exception_message(ex)) def infer(use_cuda, inference_program, params_dirname=None): From b5de0166ebd0b03a25c610ea08a0c7d5a14e665e Mon Sep 17 00:00:00 2001 From: minqiyang Date: Thu, 6 Sep 2018 13:47:29 +0800 Subject: [PATCH 07/28] Pass the INFERENCE CI when WITH_INFERENCE is OFF --- paddle/scripts/paddle_build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh index 84f9d6671a..2f12411659 100755 --- a/paddle/scripts/paddle_build.sh +++ b/paddle/scripts/paddle_build.sh @@ -554,7 +554,7 @@ function gen_capi_package() { function gen_fluid_inference_lib() { mkdir -p ${PADDLE_ROOT}/build cd ${PADDLE_ROOT}/build - if [ ${WITH_C_API:-OFF} == "OFF" ] ; then + if [[ ${WITH_C_API:-OFF} == "OFF" && ${WITH_INFERENCE:-ON} == "ON" ]] ; then cat < Date: Thu, 6 Sep 2018 13:51:18 +0800 Subject: [PATCH 08/28] Add compat deps --- .../image_classification/test_image_classification_vgg.py | 1 + .../high-level-api/recognize_digits/test_recognize_digits_mlp.py | 1 + 2 files changed, 2 insertions(+) diff --git a/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_vgg.py b/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_vgg.py index 2162989cfb..93a7215410 100644 --- a/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_vgg.py +++ b/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_vgg.py @@ -109,6 +109,7 @@ def train(use_cuda, train_program, parallel, params_dirname): feed_order=['pixel', 'label']) else: import paddle.fluid.core as core + import paddle.compat as cpt try: trainer.train( reader=train_reader, diff --git a/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_mlp.py b/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_mlp.py index f318964346..d3e4244621 100644 --- a/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_mlp.py +++ b/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_mlp.py @@ -92,6 +92,7 @@ def train(use_cuda, train_program, params_dirname, parallel): feed_order=['img', 'label']) else: import paddle.fluid.core as core + import paddle.compat as cpt try: trainer.train( num_epochs=1, From ad9e6476ff55dc7eb91fda177b7be1a83c9c21c1 Mon Sep 17 00:00:00 2001 From: minqiyang Date: Thu, 6 Sep 2018 14:47:41 +0800 Subject: [PATCH 09/28] Force object deletion on trainer in unit test --- .../test_image_classification_vgg.py | 27 +++++++------------ .../test_recognize_digits_mlp.py | 25 ++++++----------- 2 files changed, 17 insertions(+), 35 deletions(-) diff --git a/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_vgg.py b/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_vgg.py index 93a7215410..dbd8e5a881 100644 --- a/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_vgg.py +++ b/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_vgg.py @@ -14,11 +14,11 @@ from __future__ import print_function -import six import paddle import paddle.fluid as fluid import paddle.fluid.core as core import numpy +import six import os import cifar10_small_test_set @@ -101,23 +101,14 @@ def train(use_cuda, train_program, parallel, params_dirname): optimizer_func=optimizer_func, parallel=parallel) - if six.PY2: - trainer.train( - reader=train_reader, - num_epochs=1, - event_handler=event_handler, - feed_order=['pixel', 'label']) - else: - import paddle.fluid.core as core - import paddle.compat as cpt - try: - trainer.train( - reader=train_reader, - num_epochs=1, - event_handler=event_handler, - feed_order=['pixel', 'label']) - except core.EnforceNotMet as ex: - assert ("kid scope" in cpt.get_exception_message(ex)) + trainer.train( + reader=train_reader, + num_epochs=1, + event_handler=event_handler, + feed_order=['pixel', 'label']) + + if six.PY3: + del trainer def infer(use_cuda, inference_program, parallel, params_dirname=None): diff --git a/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_mlp.py b/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_mlp.py index d3e4244621..2546fdbb71 100644 --- a/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_mlp.py +++ b/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_mlp.py @@ -84,23 +84,14 @@ def train(use_cuda, train_program, params_dirname, parallel): paddle.dataset.mnist.train(), buf_size=500), batch_size=BATCH_SIZE) - if six.PY2: - trainer.train( - num_epochs=1, - event_handler=event_handler, - reader=train_reader, - feed_order=['img', 'label']) - else: - import paddle.fluid.core as core - import paddle.compat as cpt - try: - trainer.train( - num_epochs=1, - event_handler=event_handler, - reader=train_reader, - feed_order=['img', 'label']) - except core.EnforceNotMet as ex: - assert ("kid scope" in cpt.get_exception_message(ex)) + trainer.train( + num_epochs=1, + event_handler=event_handler, + reader=train_reader, + feed_order=['img', 'label']) + + if six.PY3: + del trainer def infer(use_cuda, inference_program, parallel, params_dirname=None): From 6b104c90d353409c2aacd34321bc6cf5407eb0e5 Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Thu, 6 Sep 2018 13:51:34 +0800 Subject: [PATCH 10/28] fix profile --- .../inference/analysis/analyzer_lac_tester.cc | 19 +++++++------------ .../fluid/inference/api/analysis_predictor.cc | 13 +++++++++++++ 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/paddle/fluid/inference/analysis/analyzer_lac_tester.cc b/paddle/fluid/inference/analysis/analyzer_lac_tester.cc index a6e8351c4f..1df1ade25f 100644 --- a/paddle/fluid/inference/analysis/analyzer_lac_tester.cc +++ b/paddle/fluid/inference/analysis/analyzer_lac_tester.cc @@ -178,6 +178,7 @@ void TestLACPrediction(const std::string &model_path, cfg.device = 0; cfg.specify_input_name = true; cfg.enable_ir_optim = true; + cfg.ir_passes.push_back("fc_gru_fuse_pass"); predictor = CreatePaddlePredictor(cfg); } else { @@ -208,13 +209,6 @@ void TestLACPrediction(const std::string &model_path, PrintTime(timer.toc(), batch_size, repeat); // check result - if (use_analysis) { - // run once for comparion as reference - auto ref_predictor = - CreatePaddlePredictor(config); - ref_predictor->Run(input_slots, &ref_outputs_slots); - } - EXPECT_EQ(outputs_slots.size(), 1UL); auto &out = outputs_slots[0]; size_t size = std::accumulate(out.shape.begin(), out.shape.end(), 1, @@ -228,6 +222,10 @@ void TestLACPrediction(const std::string &model_path, } if (use_analysis) { + // run once for comparion as reference + auto ref_predictor = + CreatePaddlePredictor(config); + ref_predictor->Run(input_slots, &ref_outputs_slots); EXPECT_EQ(ref_outputs_slots.size(), outputs_slots.size()); auto &ref_out = ref_outputs_slots[0]; size_t ref_size = @@ -256,12 +254,9 @@ void TestLACPrediction(const std::string &model_path, } LOG(INFO) << "has num ops: " << num_ops; ASSERT_TRUE(fuse_statis.count("fc_fuse")); - ASSERT_TRUE(fuse_statis.count("fc_gru_fuse")); - LOG(INFO) << "fc fuse num:" << fuse_statis.at("fc_fuse"); - LOG(INFO) << "fc gru fuse num:" << fuse_statis.at("fc_gru_fuse"); - // ASSERT_TRUE(fuse_statis.count("fc_gru_fuse")); - // LOG(INFO) << fuse_statis.at("fc_gru_fuse"); + LOG(INFO) << "fc fuse num:" << fuse_statis.at("fc_fuse"); + // LOG(INFO) << "fc gru fuse num:" << fuse_statis.at("fc_gru_fuse"); } } diff --git a/paddle/fluid/inference/api/analysis_predictor.cc b/paddle/fluid/inference/api/analysis_predictor.cc index a8fa677202..82d673fd15 100644 --- a/paddle/fluid/inference/api/analysis_predictor.cc +++ b/paddle/fluid/inference/api/analysis_predictor.cc @@ -22,12 +22,25 @@ #include "paddle/fluid/inference/api/paddle_inference_api.h" #include "paddle/fluid/inference/api/paddle_inference_pass.h" #include "paddle/fluid/inference/utils/singleton.h" +#include "paddle/fluid/platform/profiler.h" + +DECLARE_bool(profile); namespace paddle { bool AnalysisPredictor::Init( const std::shared_ptr& parent_scope) { VLOG(3) << "Predictor::init()"; +#if !defined(_WIN32) + if (FLAGS_profile) { + LOG(WARNING) << "Profiler is actived, might affect the performance"; + LOG(INFO) << "You can turn off by set gflags '-profile false'"; + auto tracking_device = config_.use_gpu ? platform::ProfilerState::kAll + : platform::ProfilerState::kCPU; + platform::EnableProfiler(tracking_device); + } +#endif + if (config_.use_gpu) { place_ = paddle::platform::CUDAPlace(config_.device); LOG(WARNING) << "ir optimize only supports CPU currently"; From ca30127e0a048de5e56e249d54d8836422ac2140 Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Fri, 7 Sep 2018 00:03:13 +0800 Subject: [PATCH 11/28] fix compile error undef registrar pass --- paddle/fluid/inference/analysis/analyzer.h | 1 - paddle/fluid/inference/api/CMakeLists.txt | 14 +++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/paddle/fluid/inference/analysis/analyzer.h b/paddle/fluid/inference/analysis/analyzer.h index 6189548a7b..399afbe64a 100644 --- a/paddle/fluid/inference/analysis/analyzer.h +++ b/paddle/fluid/inference/analysis/analyzer.h @@ -38,7 +38,6 @@ limitations under the License. */ #include #include #include - #include "paddle/fluid/inference/analysis/analysis_pass.h" #include "paddle/fluid/inference/analysis/flags.h" #include "paddle/fluid/inference/analysis/pass_manager.h" diff --git a/paddle/fluid/inference/api/CMakeLists.txt b/paddle/fluid/inference/api/CMakeLists.txt index b69948f40a..f944c9fdec 100644 --- a/paddle/fluid/inference/api/CMakeLists.txt +++ b/paddle/fluid/inference/api/CMakeLists.txt @@ -44,7 +44,19 @@ function(inference_api_test TARGET_NAME) endfunction(inference_api_test) cc_library(paddle_inference_api SRCS api.cc api_impl.cc helper.cc DEPS lod_tensor) -cc_library(analysis_predictor SRCS analysis_predictor.cc DEPS paddle_inference_api analysis) +cc_library(analysis_predictor SRCS analysis_predictor.cc DEPS paddle_inference_api + analysis + ir_pass_manager + pass + fc_fuse_pass + fc_lstm_fuse_pass + fc_gru_fuse_pass + seq_concat_fc_fuse_pass + graph_viz_pass + infer_clean_graph_pass + graph_pattern_detector + infer_clean_graph_pass + attention_lstm_fuse_pass) cc_test(test_paddle_inference_api SRCS api_tester.cc From 2bb0ac927b006ed375322fbd5fe4b0cbc72f39fa Mon Sep 17 00:00:00 2001 From: minqiyang Date: Fri, 7 Sep 2018 09:55:17 +0800 Subject: [PATCH 12/28] Polish code --- .../image_classification/test_image_classification_vgg.py | 8 +++++--- .../recognize_digits/test_recognize_digits_mlp.py | 8 +++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_vgg.py b/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_vgg.py index dbd8e5a881..548ebd6710 100644 --- a/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_vgg.py +++ b/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_vgg.py @@ -107,8 +107,7 @@ def train(use_cuda, train_program, parallel, params_dirname): event_handler=event_handler, feed_order=['pixel', 'label']) - if six.PY3: - del trainer + return trainer def infer(use_cuda, inference_program, parallel, params_dirname=None): @@ -132,12 +131,15 @@ def main(use_cuda, parallel): save_path = "image_classification_vgg.inference.model" os.environ['CPU_NUM'] = str(4) - train( + trainer = train( use_cuda=use_cuda, train_program=train_network, params_dirname=save_path, parallel=parallel) + if six.PY3: + del trainer + # FIXME(zcd): in the inference stage, the number of # input data is one, it is not appropriate to use parallel. if parallel and use_cuda: diff --git a/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_mlp.py b/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_mlp.py index 2546fdbb71..1e1069d5f6 100644 --- a/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_mlp.py +++ b/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_mlp.py @@ -90,8 +90,7 @@ def train(use_cuda, train_program, params_dirname, parallel): reader=train_reader, feed_order=['img', 'label']) - if six.PY3: - del trainer + return trainer def infer(use_cuda, inference_program, parallel, params_dirname=None): @@ -117,12 +116,15 @@ def main(use_cuda, parallel): # call train() with is_local argument to run distributed train os.environ['CPU_NUM'] = str(4) - train( + trainer = train( use_cuda=use_cuda, train_program=train_program, params_dirname=params_dirname, parallel=parallel) + if six.PY3: + del trainer + # FIXME(zcd): in the inference stage, the number of # input data is one, it is not appropriate to use parallel. if parallel and use_cuda: From 7eebb905235c5780350d92902776a4e0c267c87f Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Fri, 7 Sep 2018 11:53:19 +0800 Subject: [PATCH 13/28] fix conflicts --- paddle/fluid/inference/analysis/analyzer_lac_tester.cc | 2 +- paddle/fluid/inference/api/helper.h | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/paddle/fluid/inference/analysis/analyzer_lac_tester.cc b/paddle/fluid/inference/analysis/analyzer_lac_tester.cc index 5740faa746..7917152428 100644 --- a/paddle/fluid/inference/analysis/analyzer_lac_tester.cc +++ b/paddle/fluid/inference/analysis/analyzer_lac_tester.cc @@ -192,7 +192,7 @@ void TestLACPrediction(const std::string &model_path, sum += timer.toc(); } } - PrintTime(sum, batch_size, repeat); + PrintTime(batch_size, repeat, 1, 0, sum / batch_size); return; } timer.tic(); diff --git a/paddle/fluid/inference/api/helper.h b/paddle/fluid/inference/api/helper.h index 2c2ac656e8..0ab2542f34 100644 --- a/paddle/fluid/inference/api/helper.h +++ b/paddle/fluid/inference/api/helper.h @@ -124,9 +124,11 @@ std::string DescribeTensor(const PaddleTensor &tensor) { void PrintTime(int batch_size, int repeat, int num_threads, int tid, double latency) { + LOG(INFO) << "====================================="; LOG(INFO) << "batch_size: " << batch_size << ", repeat: " << repeat << ", threads: " << num_threads << ", thread id: " << tid << ", latency: " << latency << "ms"; + LOG(INFO) << "====================================="; } } // namespace inference From d4c3fe9a44503263a5560ab1ddf2ccebd17ed79e Mon Sep 17 00:00:00 2001 From: luotao1 Date: Fri, 7 Sep 2018 11:57:39 +0800 Subject: [PATCH 14/28] clean api_anakin_engine_rnn_tester --- .../api/api_anakin_engine_rnn_tester.cc | 94 +++---------------- 1 file changed, 13 insertions(+), 81 deletions(-) diff --git a/paddle/fluid/inference/api/api_anakin_engine_rnn_tester.cc b/paddle/fluid/inference/api/api_anakin_engine_rnn_tester.cc index 6183864234..98c74aaa56 100644 --- a/paddle/fluid/inference/api/api_anakin_engine_rnn_tester.cc +++ b/paddle/fluid/inference/api/api_anakin_engine_rnn_tester.cc @@ -20,71 +20,16 @@ limitations under the License. */ #include #include // NOLINT #include -#include "framework/core/net/net.h" +#include "paddle/fluid/inference/api/helper.h" #include "paddle/fluid/inference/api/paddle_inference_api.h" +#include "paddle/fluid/inference/api/timer.h" +#include "utils/logger/logger.h" DEFINE_string(model, "", "Directory of the inference model."); DEFINE_string(datapath, "", "Path of the dataset."); DEFINE_int32(batch_size, 1, "batch size."); DEFINE_int32(repeat, 1, "Running the inference program repeat times."); -// Timer for timer -class Timer { - public: - double start; - double startu; - void tic() { - struct timeval tp; - gettimeofday(&tp, NULL); - start = tp.tv_sec; - startu = tp.tv_usec; - } - double toc() { - struct timeval tp; - gettimeofday(&tp, NULL); - double used_time_ms = - (tp.tv_sec - start) * 1000.0 + (tp.tv_usec - startu) / 1000.0; - return used_time_ms; - } -}; - -std::vector string_split(std::string in_str, - std::string delimiter) { - std::vector seq; - int found = in_str.find(delimiter); - int pre_found = -1; - while (found != std::string::npos) { - if (pre_found == -1) { - seq.push_back(in_str.substr(0, found)); - } else { - seq.push_back(in_str.substr(pre_found + delimiter.length(), - found - delimiter.length() - pre_found)); - } - pre_found = found; - found = in_str.find(delimiter, pre_found + delimiter.length()); - } - seq.push_back( - in_str.substr(pre_found + 1, in_str.length() - (pre_found + 1))); - return seq; -} -std::vector string_split( - std::string in_str, std::vector& delimiter) { // NOLINT - std::vector in; - std::vector out; - out.push_back(in_str); - for (auto del : delimiter) { - in = out; - out.clear(); - for (auto s : in) { - auto out_s = string_split(s, del); - for (auto o : out_s) { - out.push_back(o); - } - } - } - return out; -} - class Data { public: Data(std::string file_name, int batch_size) @@ -120,36 +65,24 @@ void Data::get_batch_data( week_fea.clear(); time_fea.clear(); while (_file.getline(buf, 10000)) { - std::string s = buf; - std::vector deli_vec = {":"}; - std::vector data_vec = string_split(s, deli_vec); + std::vector data_vec; + paddle::inference::split(buf, ':', &data_vec); std::vector seq; - seq = string_split(data_vec[0], {"|"}); + paddle::inference::split(data_vec[0], '|', &seq); for (auto link : seq) { - std::vector data = string_split(link, ","); std::vector vec; - for (int i = 0; i < data.size(); i++) { - vec.push_back(atof(data[i].c_str())); - } + paddle::inference::split_to_float(link, ',', &vec); fea.push_back(vec); } - std::vector week_data; - std::vector time_data; - week_data = string_split(data_vec[2], ","); std::vector vec_w; - for (int i = 0; i < week_data.size(); i++) { - vec_w.push_back(atof(week_data[i].c_str())); - } + paddle::inference::split_to_float(data_vec[2], ',', &vec_w); week_fea.push_back(vec_w); - time_data = string_split(data_vec[1], ","); std::vector vec_t; - for (int i = 0; i < time_data.size(); i++) { - vec_t.push_back(atof(time_data[i].c_str())); - } + paddle::inference::split_to_float(data_vec[1], ',', &vec_t); time_fea.push_back(vec_t); cum += seq.size(); @@ -275,14 +208,13 @@ void single_test() { inputs.push_back(tensor_2); inputs.push_back(tensor_0); - Timer timer; + paddle::inference::Timer timer; timer.tic(); for (int i = 0; i < FLAGS_repeat; i++) predictor->Run(inputs, &outputs); - LOG(INFO) << "batch_size = " << FLAGS_batch_size - << ", repeat = " << FLAGS_repeat - << ", sequence_length = " << seq_offset[seq_offset.size() - 1] - << ", latency: " << timer.toc() / FLAGS_repeat << "ms"; + paddle::inference::PrintTime(FLAGS_batch_size, FLAGS_repeat, 1, 0, + timer.toc() / FLAGS_repeat); + LOG(INFO) << "sequence_length = " << seq_offset[seq_offset.size() - 1]; float* data_o = static_cast(outputs[0].data.data()); VLOG(3) << "outputs[0].data.length() = " << outputs[0].data.length(); From c9bd2d50f1d9c0db255ebc132b7c74438f3b3bba Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Fri, 7 Sep 2018 12:51:36 +0800 Subject: [PATCH 15/28] refine fc and gru pattern --- .../framework/ir/graph_pattern_detector.cc | 45 +++++++++---------- 1 file changed, 20 insertions(+), 25 deletions(-) diff --git a/paddle/fluid/framework/ir/graph_pattern_detector.cc b/paddle/fluid/framework/ir/graph_pattern_detector.cc index 37566b7621..69a323a8bd 100644 --- a/paddle/fluid/framework/ir/graph_pattern_detector.cc +++ b/paddle/fluid/framework/ir/graph_pattern_detector.cc @@ -519,50 +519,41 @@ bool VarLinksFromOp(Node* node, const std::string& op_type) { PDNode* patterns::FC(PDPattern* pattern, const std::string& name_scope, PDNode* x, bool with_bias) { - // Create Operators - PDNode* elementwise_add_op{nullptr}; + // mul op auto* mul_op = pattern->NewNode(name_scope, "mul")->assert_is_op("mul"); - if (with_bias) { - elementwise_add_op = pattern->NewNode(name_scope, "elementwise_add") - ->assert_is_op("elementwise_add"); - } - // Create variables - // w auto* mul_weight_var = pattern->NewNode(name_scope, "w") ->AsInput() ->assert_is_persistable_var() - ->assert_is_op_nth_input("mul", "Y", 0); - PDNode* mul_out_var{nullptr}; + ->assert_is_op_input("mul", "Y"); + + PDNode* fc_out{nullptr}; if (with_bias) { + PDNode* elementwise_add_op{nullptr}; + PDNode *mul_out_var{nullptr}, *bias{nullptr}; + elementwise_add_op = pattern->NewNode(name_scope, "elementwise_add") + ->assert_is_op("elementwise_add"); // intermediate variable, will be removed in the IR after fuse. mul_out_var = pattern->NewNode(name_scope, "mul_out") ->AsIntermediate() ->assert_is_only_output_of_op("mul") - ->assert_is_op_input("elementwise_add"); - } - PDNode *bias{nullptr}, *fc_out{nullptr}; - if (with_bias) { + ->assert_is_op_input("elementwise_add", "X"); // bias bias = pattern->NewNode(name_scope, "fc_bias") - ->assert_is_op_input("elementwise_add") - ->AsInput(); + ->AsInput() + ->assert_is_persistable_var() + ->assert_is_op_input("elementwise_add", "Y"); // output fc_out = pattern->NewNode(name_scope, "fc_out") ->AsOutput() - ->assert_is_op_output("elementwise_add"); + ->assert_is_op_output("elementwise_add", "Out"); + mul_op->LinksFrom({x, mul_weight_var}).LinksTo({mul_out_var}); + elementwise_add_op->LinksFrom({mul_out_var, bias}).LinksTo({fc_out}); } else { fc_out = pattern->NewNode(name_scope, "fc_out") ->AsOutput() - ->assert_is_op_output("mul"); - } - - if (with_bias) { - mul_op->LinksFrom({mul_weight_var, x}).LinksTo({mul_out_var}); - elementwise_add_op->LinksFrom({mul_out_var, bias}).LinksTo({fc_out}); - } else { + ->assert_is_op_output("mul", "Out"); mul_op->LinksFrom({mul_weight_var, x}).LinksTo({fc_out}); } - return fc_out; } @@ -609,6 +600,10 @@ PDNode* patterns::GRU(PDPattern* pattern, const std::string& name_scope, NEW_NODE(gru, BatchResetHiddenPrev, output); NEW_NODE(gru, BatchHidden, output); + BatchGate->AsIntermediate(); + BatchResetHiddenPrev->AsIntermediate(); + BatchHidden->AsIntermediate(); + gru_op->LinksFrom({x, Weight, Bias}); gru_op->LinksTo({Hidden, BatchGate, BatchResetHiddenPrev, BatchHidden}); return Hidden; From 8ef1f9f9b837639737bb8d3e119eeb04f3c4c303 Mon Sep 17 00:00:00 2001 From: minqiyang Date: Fri, 7 Sep 2018 14:22:44 +0800 Subject: [PATCH 16/28] Polish code --- .../test_image_classification_vgg.py | 11 ++++++----- .../recognize_digits/test_recognize_digits_mlp.py | 11 ++++++----- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_vgg.py b/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_vgg.py index 548ebd6710..2767e8b5d9 100644 --- a/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_vgg.py +++ b/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_vgg.py @@ -107,7 +107,11 @@ def train(use_cuda, train_program, parallel, params_dirname): event_handler=event_handler, feed_order=['pixel', 'label']) - return trainer + def _del_trainer(trainer): + del trainer + + if six.PY3: + _del_trainer(trainer) def infer(use_cuda, inference_program, parallel, params_dirname=None): @@ -131,15 +135,12 @@ def main(use_cuda, parallel): save_path = "image_classification_vgg.inference.model" os.environ['CPU_NUM'] = str(4) - trainer = train( + train( use_cuda=use_cuda, train_program=train_network, params_dirname=save_path, parallel=parallel) - if six.PY3: - del trainer - # FIXME(zcd): in the inference stage, the number of # input data is one, it is not appropriate to use parallel. if parallel and use_cuda: diff --git a/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_mlp.py b/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_mlp.py index 1e1069d5f6..b784657466 100644 --- a/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_mlp.py +++ b/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_mlp.py @@ -90,7 +90,11 @@ def train(use_cuda, train_program, params_dirname, parallel): reader=train_reader, feed_order=['img', 'label']) - return trainer + def _del_trainer(trainer): + del trainer + + if six.PY3: + _del_trainer(trainer) def infer(use_cuda, inference_program, parallel, params_dirname=None): @@ -116,15 +120,12 @@ def main(use_cuda, parallel): # call train() with is_local argument to run distributed train os.environ['CPU_NUM'] = str(4) - trainer = train( + train( use_cuda=use_cuda, train_program=train_program, params_dirname=params_dirname, parallel=parallel) - if six.PY3: - del trainer - # FIXME(zcd): in the inference stage, the number of # input data is one, it is not appropriate to use parallel. if parallel and use_cuda: From 2720330dc57fd54ea1e27b0c10801089da1cf738 Mon Sep 17 00:00:00 2001 From: minqiyang Date: Fri, 7 Sep 2018 14:25:25 +0800 Subject: [PATCH 17/28] Disable random fail case --- .../test_image_classification_vgg.py | 9 ++------- .../recognize_digits/test_recognize_digits_mlp.py | 9 ++------- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_vgg.py b/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_vgg.py index 2767e8b5d9..d4e6742b78 100644 --- a/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_vgg.py +++ b/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_vgg.py @@ -107,12 +107,6 @@ def train(use_cuda, train_program, parallel, params_dirname): event_handler=event_handler, feed_order=['pixel', 'label']) - def _del_trainer(trainer): - del trainer - - if six.PY3: - _del_trainer(trainer) - def infer(use_cuda, inference_program, parallel, params_dirname=None): place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() @@ -158,4 +152,5 @@ if __name__ == '__main__': for parallel in (False, True): if use_cuda and not core.is_compiled_with_cuda(): continue - main(use_cuda=use_cuda, parallel=parallel) + if six.PY2: + main(use_cuda=use_cuda, parallel=parallel) diff --git a/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_mlp.py b/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_mlp.py index b784657466..811f17b5ab 100644 --- a/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_mlp.py +++ b/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_mlp.py @@ -90,12 +90,6 @@ def train(use_cuda, train_program, params_dirname, parallel): reader=train_reader, feed_order=['img', 'label']) - def _del_trainer(trainer): - del trainer - - if six.PY3: - _del_trainer(trainer) - def infer(use_cuda, inference_program, parallel, params_dirname=None): place = fluid.CUDAPlace(0) if use_cuda else fluid.CPUPlace() @@ -143,4 +137,5 @@ if __name__ == '__main__': for parallel in (False, True): if use_cuda and not core.is_compiled_with_cuda(): continue - main(use_cuda=use_cuda, parallel=parallel) + if six.PY2: + main(use_cuda=use_cuda, parallel=parallel) From df0c695618696378c8320dd85661fdaa276e7407 Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Fri, 7 Sep 2018 12:53:15 +0800 Subject: [PATCH 18/28] fix fusion gru pass and enable it --- paddle/fluid/framework/ir/fc_gru_fuse_pass.cc | 98 +++++++++++-------- .../inference/analysis/analyzer_lac_tester.cc | 1 - 2 files changed, 56 insertions(+), 43 deletions(-) diff --git a/paddle/fluid/framework/ir/fc_gru_fuse_pass.cc b/paddle/fluid/framework/ir/fc_gru_fuse_pass.cc index 4a08beee7d..90d8d5c042 100644 --- a/paddle/fluid/framework/ir/fc_gru_fuse_pass.cc +++ b/paddle/fluid/framework/ir/fc_gru_fuse_pass.cc @@ -28,7 +28,7 @@ static void BuildPattern(PDPattern* pattern, const std::string& name_scope, auto* fc_out = patterns::FC(pattern, name_scope, x, with_fc_bias); fc_out->AsIntermediate(); // fc_out is a tmp var, will be removed after fuse. patterns::GRU(pattern, name_scope, fc_out); - VLOG(3) << "\n" << pattern->DotString(); + VLOG(3) << "fc_gru pattern \n" << pattern->DotString(); } static int BuildFusion(Graph* graph, const std::string& name_scope, @@ -51,65 +51,72 @@ static int BuildFusion(Graph* graph, const std::string& name_scope, OpDesc op_desc; op_desc.SetType("fusion_gru"); + +#define NEW_NAME(x) name_scope + "/at." #x ".new" #define SET_IN(Key, node__) op_desc.SetInput(#Key, {node__##_n->Name()}); SET_IN(X, x); SET_IN(WeightX, weight_x); SET_IN(WeightH, weight_h); - SET_IN(Bias, bias); + if (with_fc_bias) { + op_desc.SetInput("Bias", {NEW_NAME(bias) + bias_n->Name()}); + } else { + SET_IN(Bias, bias); + } #undef SET_IN + op_desc.SetInput("H0", {}); + op_desc.SetOutput("Hidden", {hidden_n->Name()}); + op_desc.SetAttr("is_reverse", gru_n->Op()->GetAttr("is_reverse")); + // TODO(TJ): This should be a option for infer + op_desc.SetAttr("use_seq", true); + +#define SET_IMTERMEDIATE_OUT(key) op_desc.SetOutput(#key, {NEW_NAME(key)}) + SET_IMTERMEDIATE_OUT(ReorderedH0); + SET_IMTERMEDIATE_OUT(XX); + SET_IMTERMEDIATE_OUT(BatchedInput); + SET_IMTERMEDIATE_OUT(BatchedOut); +#undef SET_IMTERMEDIATE_OUT + + auto* op = graph->CreateOpNode(&op_desc); + PADDLE_ENFORCE(graph->Has(kParamScopeAttr)); + auto* scope = graph->Get(kParamScopeAttr); + PADDLE_ENFORCE(scope); if (with_fc_bias) { - // Add FC-bias with LSTM-bias and create a new weight - PADDLE_ENFORCE(scope); - const std::string& new_bias_var = name_scope + "_bias.new"; - auto* bias_var = scope->Var(new_bias_var); - PADDLE_ENFORCE(bias_var); - auto* bias_tensor = bias_var->GetMutable(); + // Fusion GRU bias = fcbias + grubias + auto* fusion_bias_var = scope->Var(NEW_NAME(bias) + bias_n->Name()); + auto* out_bias_tensor = + fusion_bias_var->GetMutable(); + PADDLE_ENFORCE(fusion_bias_var); + GET_NODE(fc_bias); + PADDLE_ENFORCE(fc_bias_n); auto* gru_bias_var = scope->FindVar(bias_n->Name()); + auto* fc_bias_var = scope->FindVar(fc_bias_n->Name()); PADDLE_ENFORCE(gru_bias_var); + PADDLE_ENFORCE(fc_bias_var); const auto& gru_bias_tenosr = gru_bias_var->Get(); - bias_tensor->Resize(gru_bias_tenosr.dims()); - - GET_NODE(fc_bias); - auto* fc_bias_var = scope->FindVar(fc_bias_n->Name()); const auto& fc_bias_tensor = fc_bias_var->Get(); // new bias = fc bias + gru bias - auto* data = bias_tensor->mutable_data(platform::CPUPlace()); - for (int i = 0; i < bias_tensor->numel(); i++) { + out_bias_tensor->Resize(gru_bias_tenosr.dims()); + auto* data = out_bias_tensor->mutable_data(platform::CPUPlace()); + for (int i = 0; i < out_bias_tensor->numel(); i++) { data[i] = fc_bias_tensor.data()[i] + gru_bias_tenosr.data()[i]; } - op_desc.SetInput("Bias", {new_bias_var}); } #undef GET_NODE - op_desc.SetInput("H0", {}); - op_desc.SetOutput("Hidden", {hidden_n->Name()}); - op_desc.SetAttr("is_reverse", gru_n->Op()->GetAttr("is_reverse")); - // TODO(TJ): This should be a option for infer - op_desc.SetAttr("use_seq", true); - - // Create temp variables. - // TODO(TJ): clean code - scope->Var(name_scope + "/ReorderedH0.new") - ->GetMutable(); - scope->Var(name_scope + "/XX.new")->GetMutable(); - scope->Var(name_scope + "/BatchedInput.new") - ->GetMutable(); - scope->Var(name_scope + "/BatchedOut.new") - ->GetMutable(); - op_desc.SetOutput("ReorderedH0", {name_scope + "/ReorderedH0.new"}); - op_desc.SetOutput("XX", {name_scope + "/XX.new"}); - op_desc.SetOutput("BatchedInput", {name_scope + "/BatchedInput.new"}); - op_desc.SetOutput("BatchedOut", {name_scope + "/BatchedOut.new"}); - - auto* op = graph->CreateOpNode(&op_desc); - PADDLE_ENFORCE(graph->Has(kParamScopeAttr)); - // auto* scope = graph->Get(kParamScopeAttr); +#define NEW_IMTERMEDIATE_OUT(key) \ + scope->Var(NEW_NAME(key))->GetMutable() + NEW_IMTERMEDIATE_OUT(ReorderedH0); + NEW_IMTERMEDIATE_OUT(XX); + NEW_IMTERMEDIATE_OUT(BatchedInput); + NEW_IMTERMEDIATE_OUT(BatchedOut); +#undef NEW_NAME +#undef NEW_IMTERMEDIATE_OUT IR_NODE_LINK_TO(x_n, op); IR_NODE_LINK_TO(weight_x_n, op); IR_NODE_LINK_TO(weight_h_n, op); - IR_NODE_LINK_TO(bias_n, op); + IR_NODE_LINK_TO(bias_n, op); // actually should link to new bias if have IR_NODE_LINK_TO(op, hidden_n); // h0? return op; @@ -127,26 +134,33 @@ static int BuildFusion(Graph* graph, const std::string& name_scope, int name__ __attribute__((unused)) = name__##_n->id(); GET_NODE(x); - GET_NODE(w); + GET_NODE(w); // fc weight GET_NODE(mul); GET_NODE(fc_out); GET_NODE(Weight); GET_NODE(gru); GET_NODE(Bias); GET_NODE(Hidden); + // nodes need be removed + GET_NODE(BatchGate); + GET_NODE(BatchResetHiddenPrev); + GET_NODE(BatchHidden); if (with_fc_bias) { + GET_NODE(mul_out); GET_NODE(fc_bias); GET_NODE(elementwise_add); gru_creater(gru, x, w, Weight, Bias, Hidden, fc_bias); // Remove unneeded nodes. std::unordered_set marked_nodes( - {mul_n, gru_n, elementwise_add_n}); + {mul_n, gru_n, elementwise_add_n, fc_bias_n, fc_out_n, mul_out_n, + BatchGate_n, BatchResetHiddenPrev_n, BatchHidden_n}); GraphSafeRemoveNodes(graph, marked_nodes); } else { gru_creater(gru, x, w, Weight, Bias, Hidden, -1); // Remove unneeded nodes. - std::unordered_set marked_nodes({mul_n, gru_n}); + std::unordered_set marked_nodes( + {mul_n, gru_n, BatchGate_n, BatchResetHiddenPrev_n, BatchHidden_n}); GraphSafeRemoveNodes(graph, marked_nodes); } #undef GET_NODE diff --git a/paddle/fluid/inference/analysis/analyzer_lac_tester.cc b/paddle/fluid/inference/analysis/analyzer_lac_tester.cc index 7917152428..56f773bf21 100644 --- a/paddle/fluid/inference/analysis/analyzer_lac_tester.cc +++ b/paddle/fluid/inference/analysis/analyzer_lac_tester.cc @@ -171,7 +171,6 @@ void TestLACPrediction(const std::string &model_path, cfg.device = 0; cfg.specify_input_name = true; cfg.enable_ir_optim = true; - cfg.ir_passes.push_back("fc_gru_fuse_pass"); predictor = CreatePaddlePredictor(cfg); } else { From 5335ff628e1494a6a7a6583901ab75f85600e402 Mon Sep 17 00:00:00 2001 From: minqiyang Date: Fri, 7 Sep 2018 15:30:26 +0800 Subject: [PATCH 19/28] Polish code --- .../image_classification/test_image_classification_vgg.py | 2 ++ .../recognize_digits/test_recognize_digits_conv.py | 5 ++++- .../recognize_digits/test_recognize_digits_mlp.py | 2 ++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_vgg.py b/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_vgg.py index d4e6742b78..ff91be72c9 100644 --- a/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_vgg.py +++ b/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_vgg.py @@ -152,5 +152,7 @@ if __name__ == '__main__': for parallel in (False, True): if use_cuda and not core.is_compiled_with_cuda(): continue + # TODO(minqiyang): remove this line after fixing the deletion + # order problem of Scope in ParallelExecutor in manylinux if six.PY2: main(use_cuda=use_cuda, parallel=parallel) diff --git a/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_conv.py b/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_conv.py index a5adf68158..df4c721c4e 100644 --- a/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_conv.py +++ b/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_conv.py @@ -154,4 +154,7 @@ if __name__ == '__main__': for parallel in (False, True): if use_cuda and not core.is_compiled_with_cuda(): continue - main(use_cuda=use_cuda, parallel=parallel) + # TODO(minqiyang): remove this line after fixing the deletion + # order problem of Scope in ParallelExecutor in manylinux + if six.PY2: + main(use_cuda=use_cuda, parallel=parallel) diff --git a/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_mlp.py b/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_mlp.py index 811f17b5ab..440d2a3083 100644 --- a/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_mlp.py +++ b/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_mlp.py @@ -137,5 +137,7 @@ if __name__ == '__main__': for parallel in (False, True): if use_cuda and not core.is_compiled_with_cuda(): continue + # TODO(minqiyang): remove this line after fixing the deletion + # order problem of Scope in ParallelExecutor in manylinux if six.PY2: main(use_cuda=use_cuda, parallel=parallel) From 6fae46a128440b5664845f0a47dc8df1623f995e Mon Sep 17 00:00:00 2001 From: luotao1 Date: Fri, 7 Sep 2018 16:05:31 +0800 Subject: [PATCH 20/28] refine codes --- cmake/external/anakin.cmake | 2 +- paddle/scripts/paddle_build.sh | 2 +- python/paddle/fluid/layers/nn.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/external/anakin.cmake b/cmake/external/anakin.cmake index 5a12c6490e..ed054ff41a 100644 --- a/cmake/external/anakin.cmake +++ b/cmake/external/anakin.cmake @@ -48,7 +48,7 @@ ExternalProject_Add( ${EXTERNAL_PROJECT_LOG_ARGS} DEPENDS ${MKLML_PROJECT} GIT_REPOSITORY "https://github.com/PaddlePaddle/Anakin" - GIT_TAG "9424277cf9ae180a14aff09560d3cd60a49c76d2" + GIT_TAG "3c8554f4978628183566ab7dd6c1e7e66493c7cd" PREFIX ${ANAKIN_SOURCE_DIR} UPDATE_COMMAND "" CMAKE_ARGS ${CMAKE_ARGS_PREFIX} diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh index 9ffde5df96..69f5ffecbe 100755 --- a/paddle/scripts/paddle_build.sh +++ b/paddle/scripts/paddle_build.sh @@ -547,7 +547,7 @@ function gen_capi_package() { rm -rf $install_prefix make DESTDIR="$install_prefix" install cd $install_prefix/usr/local - ls | egrep -v "^Found.*item$" | xargs tar -cf ${PADDLE_ROOT}/build/paddle.tgz + ls | egrep -v "^Found.*item$" | xargs tar -czf ${PADDLE_ROOT}/build/paddle.tgz fi } diff --git a/python/paddle/fluid/layers/nn.py b/python/paddle/fluid/layers/nn.py index 5f49d5bbff..8408e6d2a1 100644 --- a/python/paddle/fluid/layers/nn.py +++ b/python/paddle/fluid/layers/nn.py @@ -4500,7 +4500,7 @@ def reshape(x, shape, actual_shape=None, act=None, inplace=True, name=None): """ if not (isinstance(shape, list) or isinstance(shape, tuple)): - raise ValueError("Input shape must be a python lsit or tuple.") + raise ValueError("Input shape must be a python list or tuple.") inputs = {"X": x} if isinstance(actual_shape, Variable): inputs["Shape"] = actual_shape From acfdbf029330e60037e4fff7cee9c00d99f031c5 Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Fri, 7 Sep 2018 15:56:51 +0800 Subject: [PATCH 21/28] enable ner analysis test and refine lac --- .../fluid/inference/analysis/CMakeLists.txt | 2 +- .../inference/analysis/analyzer_lac_tester.cc | 14 ++-- .../inference/analysis/analyzer_ner_tester.cc | 74 ++++++++++++++++--- .../inference/analysis/analyzer_tester.cc | 2 - paddle/fluid/inference/api/CMakeLists.txt | 15 +--- paddle/fluid/inference/api/helper.h | 6 +- 6 files changed, 74 insertions(+), 39 deletions(-) diff --git a/paddle/fluid/inference/analysis/CMakeLists.txt b/paddle/fluid/inference/analysis/CMakeLists.txt index ef55a0c28a..a115bc8f4a 100644 --- a/paddle/fluid/inference/analysis/CMakeLists.txt +++ b/paddle/fluid/inference/analysis/CMakeLists.txt @@ -81,7 +81,7 @@ if (NOT EXISTS ${CHINESE_NER_INSTALL_DIR} AND WITH_TESTING AND WITH_INFERENCE) endif() inference_analysis_test(test_analyzer_ner SRCS analyzer_ner_tester.cc - EXTRA_DEPS paddle_inference_api paddle_fluid_api + EXTRA_DEPS paddle_inference_api paddle_fluid_api analysis_predictor ARGS --infer_model=${CHINESE_NER_INSTALL_DIR}/model --infer_data=${CHINESE_NER_INSTALL_DIR}/data.txt) diff --git a/paddle/fluid/inference/analysis/analyzer_lac_tester.cc b/paddle/fluid/inference/analysis/analyzer_lac_tester.cc index 56f773bf21..4ff7251473 100644 --- a/paddle/fluid/inference/analysis/analyzer_lac_tester.cc +++ b/paddle/fluid/inference/analysis/analyzer_lac_tester.cc @@ -15,11 +15,9 @@ #include "paddle/fluid/inference/analysis/analyzer.h" #include #include "paddle/fluid/framework/ir/fuse_pass_base.h" -#include "paddle/fluid/framework/ir/pass.h" -#include "paddle/fluid/inference/analysis/ut_helper.h" #include "paddle/fluid/inference/api/analysis_predictor.h" #include "paddle/fluid/inference/api/helper.h" -#include "paddle/fluid/inference/api/paddle_inference_api.h" +#include "paddle/fluid/inference/api/paddle_inference_pass.h" #include "paddle/fluid/platform/profiler.h" DEFINE_string(infer_model, "", "model path for LAC"); @@ -160,7 +158,7 @@ void TestLACPrediction(const std::string &model_path, config.use_gpu = false; config.device = 0; config.specify_input_name = true; - std::vector input_slots, outputs_slots, ref_outputs_slots; + std::vector input_slots, outputs_slots; DataRecord data(data_file, batch_size); GetOneBatch(&input_slots, &data, batch_size); std::unique_ptr predictor; @@ -217,6 +215,7 @@ void TestLACPrediction(const std::string &model_path, // run once for comparion as reference auto ref_predictor = CreatePaddlePredictor(config); + std::vector ref_outputs_slots; ref_predictor->Run(input_slots, &ref_outputs_slots); EXPECT_EQ(ref_outputs_slots.size(), outputs_slots.size()); auto &ref_out = ref_outputs_slots[0]; @@ -246,9 +245,10 @@ void TestLACPrediction(const std::string &model_path, } LOG(INFO) << "has num ops: " << num_ops; ASSERT_TRUE(fuse_statis.count("fc_fuse")); - // ASSERT_TRUE(fuse_statis.count("fc_gru_fuse")); - LOG(INFO) << "fc fuse num:" << fuse_statis.at("fc_fuse"); - // LOG(INFO) << "fc gru fuse num:" << fuse_statis.at("fc_gru_fuse"); + ASSERT_TRUE(fuse_statis.count("fc_gru_fuse")); + EXPECT_EQ(fuse_statis.at("fc_fuse"), 1); + EXPECT_EQ(fuse_statis.at("fc_gru_fuse"), 4); + EXPECT_EQ(num_ops, 11); } } diff --git a/paddle/fluid/inference/analysis/analyzer_ner_tester.cc b/paddle/fluid/inference/analysis/analyzer_ner_tester.cc index eaae09b051..f5c5d73aeb 100644 --- a/paddle/fluid/inference/analysis/analyzer_ner_tester.cc +++ b/paddle/fluid/inference/analysis/analyzer_ner_tester.cc @@ -13,12 +13,11 @@ // limitations under the License. #include "paddle/fluid/inference/analysis/analyzer.h" -#include #include -#include "paddle/fluid/framework/ir/pass.h" -#include "paddle/fluid/inference/analysis/ut_helper.h" +#include "paddle/fluid/framework/ir/fuse_pass_base.h" +#include "paddle/fluid/inference/api/analysis_predictor.h" #include "paddle/fluid/inference/api/helper.h" -#include "paddle/fluid/inference/api/paddle_inference_api.h" +#include "paddle/fluid/inference/api/paddle_inference_pass.h" #include "paddle/fluid/platform/profiler.h" DEFINE_string(infer_model, "", "model path"); @@ -112,7 +111,7 @@ void PrepareInputs(std::vector *input_slots, DataRecord *data, const int chinese_ner_result_data[] = {30, 45, 41, 48, 17, 26, 48, 39, 38, 16, 25}; -void TestChineseNERPrediction() { +void TestChineseNERPrediction(bool use_analysis) { NativeConfig config; config.prog_file = FLAGS_infer_model + "/__model__"; config.param_file = FLAGS_infer_model + "/param"; @@ -120,11 +119,23 @@ void TestChineseNERPrediction() { config.device = 0; config.specify_input_name = true; - auto predictor = - CreatePaddlePredictor(config); - std::vector input_slots; - std::vector outputs; + std::vector input_slots, outputs; + std::unique_ptr predictor; Timer timer; + if (use_analysis) { + AnalysisConfig cfg; + cfg.prog_file = FLAGS_infer_model + "/__model__"; + cfg.param_file = FLAGS_infer_model + "/param"; + cfg.use_gpu = false; + cfg.device = 0; + cfg.specify_input_name = true; + cfg.enable_ir_optim = true; + predictor = + CreatePaddlePredictor(cfg); + } else { + predictor = + CreatePaddlePredictor(config); + } if (FLAGS_test_all_data) { LOG(INFO) << "test all data"; @@ -165,10 +176,51 @@ void TestChineseNERPrediction() { for (size_t i = 0; i < std::min(11UL, size); i++) { PADDLE_ENFORCE(result[i], chinese_ner_result_data[i]); } + + if (use_analysis) { + // run once for comparion as reference + auto ref_predictor = + CreatePaddlePredictor(config); + std::vector ref_outputs_slots; + ref_predictor->Run(input_slots, &ref_outputs_slots); + EXPECT_EQ(ref_outputs_slots.size(), outputs.size()); + auto &ref_out = ref_outputs_slots[0]; + size_t ref_size = + std::accumulate(ref_out.shape.begin(), ref_out.shape.end(), 1, + [](int a, int b) { return a * b; }); + EXPECT_EQ(size, ref_size); + int64_t *pdata_ref = static_cast(ref_out.data.data()); + for (size_t i = 0; i < size; ++i) { + EXPECT_EQ(pdata_ref[i], result[i]); + } + + AnalysisPredictor *analysis_predictor = + dynamic_cast(predictor.get()); + auto &fuse_statis = analysis_predictor->analysis_argument() + .Get>( + framework::ir::kFuseStatisAttr); + for (auto &item : fuse_statis) { + LOG(INFO) << "fused " << item.first << " " << item.second; + } + int num_ops = 0; + for (auto &node : + analysis_predictor->analysis_argument().main_dfg->nodes.nodes()) { + if (node->IsFunction()) { + ++num_ops; + } + } + LOG(INFO) << "has num ops: " << num_ops; + ASSERT_TRUE(fuse_statis.count("fc_fuse")); + ASSERT_TRUE(fuse_statis.count("fc_gru_fuse")); + EXPECT_EQ(fuse_statis.at("fc_fuse"), 1); + EXPECT_EQ(fuse_statis.at("fc_gru_fuse"), 2); + EXPECT_EQ(num_ops, 14); + } } -// Directly infer with the original model. -TEST(Analyzer, Chinese_ner) { TestChineseNERPrediction(); } +TEST(Analyzer_Chinese_ner, native) { TestChineseNERPrediction(false); } + +TEST(Analyzer_Chinese_ner, analysis) { TestChineseNERPrediction(true); } } // namespace inference } // namespace paddle diff --git a/paddle/fluid/inference/analysis/analyzer_tester.cc b/paddle/fluid/inference/analysis/analyzer_tester.cc index 4cf26d3c70..a496ae41aa 100644 --- a/paddle/fluid/inference/analysis/analyzer_tester.cc +++ b/paddle/fluid/inference/analysis/analyzer_tester.cc @@ -283,7 +283,6 @@ void TestDituRNNPrediction(bool use_analysis, bool activate_ir, base_predictor->Run(input_slots, &base_outputs); - LOG(INFO) << "===========profile result==========="; if (num_threads == 1) { // Prepare inputs. Timer timer; @@ -324,7 +323,6 @@ void TestDituRNNPrediction(bool use_analysis, bool activate_ir, threads[i].join(); } } - LOG(INFO) << "====================================="; if (use_analysis && activate_ir) { AnalysisPredictor *analysis_predictor = diff --git a/paddle/fluid/inference/api/CMakeLists.txt b/paddle/fluid/inference/api/CMakeLists.txt index f944c9fdec..5df486f345 100644 --- a/paddle/fluid/inference/api/CMakeLists.txt +++ b/paddle/fluid/inference/api/CMakeLists.txt @@ -44,20 +44,7 @@ function(inference_api_test TARGET_NAME) endfunction(inference_api_test) cc_library(paddle_inference_api SRCS api.cc api_impl.cc helper.cc DEPS lod_tensor) -cc_library(analysis_predictor SRCS analysis_predictor.cc DEPS paddle_inference_api - analysis - ir_pass_manager - pass - fc_fuse_pass - fc_lstm_fuse_pass - fc_gru_fuse_pass - seq_concat_fc_fuse_pass - graph_viz_pass - infer_clean_graph_pass - graph_pattern_detector - infer_clean_graph_pass - attention_lstm_fuse_pass) - +cc_library(analysis_predictor SRCS analysis_predictor.cc DEPS paddle_inference_api analysis) cc_test(test_paddle_inference_api SRCS api_tester.cc DEPS paddle_inference_api) diff --git a/paddle/fluid/inference/api/helper.h b/paddle/fluid/inference/api/helper.h index 0ab2542f34..f6893be428 100644 --- a/paddle/fluid/inference/api/helper.h +++ b/paddle/fluid/inference/api/helper.h @@ -124,11 +124,9 @@ std::string DescribeTensor(const PaddleTensor &tensor) { void PrintTime(int batch_size, int repeat, int num_threads, int tid, double latency) { - LOG(INFO) << "====================================="; - LOG(INFO) << "batch_size: " << batch_size << ", repeat: " << repeat + LOG(INFO) << "====== batch_size: " << batch_size << ", repeat: " << repeat << ", threads: " << num_threads << ", thread id: " << tid - << ", latency: " << latency << "ms"; - LOG(INFO) << "====================================="; + << ", latency: " << latency << "ms ======"; } } // namespace inference From d3d22a12fdae2434a7aa29ce9b324ff0ddb3be96 Mon Sep 17 00:00:00 2001 From: minqiyang Date: Fri, 7 Sep 2018 17:53:41 +0800 Subject: [PATCH 22/28] Polish code --- .../recognize_digits/test_recognize_digits_conv.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_conv.py b/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_conv.py index df4c721c4e..fa72c939e5 100644 --- a/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_conv.py +++ b/python/paddle/fluid/tests/book/high-level-api/recognize_digits/test_recognize_digits_conv.py @@ -18,6 +18,7 @@ import argparse import paddle.fluid as fluid import paddle.fluid.core as core import paddle +import six import sys import numpy import unittest From d230379b542ad08501f455ca3d492d643867c8b8 Mon Sep 17 00:00:00 2001 From: luotao1 Date: Fri, 7 Sep 2018 18:18:49 +0800 Subject: [PATCH 23/28] move anakin release to third_party/install/anakin --- cmake/inference_lib.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/inference_lib.cmake b/cmake/inference_lib.cmake index 6e66ba94ab..077072f6ea 100644 --- a/cmake/inference_lib.cmake +++ b/cmake/inference_lib.cmake @@ -150,7 +150,7 @@ if (WITH_ANAKIN AND WITH_MKL) SRCS ${PADDLE_BINARY_DIR}/paddle/fluid/inference/api/libinference_anakin_api* # compiled anakin api ${ANAKIN_INSTALL_DIR} # anakin release - DSTS ${dst_dir}/inference/anakin ${dst_dir}/inference/anakin) + DSTS ${dst_dir}/inference/anakin ${FLUID_INSTALL_DIR}/third_party/install/anakin) list(APPEND inference_deps anakin_inference_lib) endif() From 3ea19b759649feabf45860e4e4c808c26845c3c7 Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Fri, 7 Sep 2018 18:48:45 +0800 Subject: [PATCH 24/28] fix bug and fc pass ut --- paddle/fluid/framework/ir/graph_pattern_detector.cc | 9 ++++----- paddle/fluid/inference/analysis/analyzer_lac_tester.cc | 1 + paddle/fluid/inference/analysis/analyzer_ner_tester.cc | 1 + 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/paddle/fluid/framework/ir/graph_pattern_detector.cc b/paddle/fluid/framework/ir/graph_pattern_detector.cc index 69a323a8bd..5ca7509515 100644 --- a/paddle/fluid/framework/ir/graph_pattern_detector.cc +++ b/paddle/fluid/framework/ir/graph_pattern_detector.cc @@ -536,22 +536,21 @@ PDNode* patterns::FC(PDPattern* pattern, const std::string& name_scope, mul_out_var = pattern->NewNode(name_scope, "mul_out") ->AsIntermediate() ->assert_is_only_output_of_op("mul") - ->assert_is_op_input("elementwise_add", "X"); + ->assert_is_op_input("elementwise_add"); // bias bias = pattern->NewNode(name_scope, "fc_bias") ->AsInput() - ->assert_is_persistable_var() - ->assert_is_op_input("elementwise_add", "Y"); + ->assert_is_op_input("elementwise_add"); // output fc_out = pattern->NewNode(name_scope, "fc_out") ->AsOutput() - ->assert_is_op_output("elementwise_add", "Out"); + ->assert_is_op_output("elementwise_add"); mul_op->LinksFrom({x, mul_weight_var}).LinksTo({mul_out_var}); elementwise_add_op->LinksFrom({mul_out_var, bias}).LinksTo({fc_out}); } else { fc_out = pattern->NewNode(name_scope, "fc_out") ->AsOutput() - ->assert_is_op_output("mul", "Out"); + ->assert_is_op_output("mul"); mul_op->LinksFrom({mul_weight_var, x}).LinksTo({fc_out}); } return fc_out; diff --git a/paddle/fluid/inference/analysis/analyzer_lac_tester.cc b/paddle/fluid/inference/analysis/analyzer_lac_tester.cc index 4ff7251473..b906b32cf5 100644 --- a/paddle/fluid/inference/analysis/analyzer_lac_tester.cc +++ b/paddle/fluid/inference/analysis/analyzer_lac_tester.cc @@ -15,6 +15,7 @@ #include "paddle/fluid/inference/analysis/analyzer.h" #include #include "paddle/fluid/framework/ir/fuse_pass_base.h" +#include "paddle/fluid/inference/analysis/ut_helper.h" #include "paddle/fluid/inference/api/analysis_predictor.h" #include "paddle/fluid/inference/api/helper.h" #include "paddle/fluid/inference/api/paddle_inference_pass.h" diff --git a/paddle/fluid/inference/analysis/analyzer_ner_tester.cc b/paddle/fluid/inference/analysis/analyzer_ner_tester.cc index f5c5d73aeb..661b047ed7 100644 --- a/paddle/fluid/inference/analysis/analyzer_ner_tester.cc +++ b/paddle/fluid/inference/analysis/analyzer_ner_tester.cc @@ -15,6 +15,7 @@ #include "paddle/fluid/inference/analysis/analyzer.h" #include #include "paddle/fluid/framework/ir/fuse_pass_base.h" +#include "paddle/fluid/inference/analysis/ut_helper.h" #include "paddle/fluid/inference/api/analysis_predictor.h" #include "paddle/fluid/inference/api/helper.h" #include "paddle/fluid/inference/api/paddle_inference_pass.h" From ce24a92007b7c55f6264e79573d7208e0c4b2628 Mon Sep 17 00:00:00 2001 From: minqiyang Date: Fri, 7 Sep 2018 19:13:44 +0800 Subject: [PATCH 25/28] Disable image_classification_resnet --- .../test_image_classification_resnet.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_resnet.py b/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_resnet.py index 2e15c224f6..e5ae95e2d9 100644 --- a/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_resnet.py +++ b/python/paddle/fluid/tests/book/high-level-api/image_classification/test_image_classification_resnet.py @@ -18,6 +18,7 @@ import paddle import paddle.fluid as fluid import paddle.fluid.core as core import numpy +import six import os import cifar10_small_test_set @@ -177,4 +178,7 @@ if __name__ == '__main__': for parallel in (False, True): if use_cuda and not core.is_compiled_with_cuda(): continue - main(use_cuda=use_cuda, parallel=parallel) + # TODO(minqiyang): remove this line after fixing the deletion + # order problem of Scope in ParallelExecutor in manylinux + if six.PY2: + main(use_cuda=use_cuda, parallel=parallel) From 5a2fc5b52f20b0c905a38ebd0fe206f88dadd649 Mon Sep 17 00:00:00 2001 From: tensor-tang Date: Fri, 7 Sep 2018 22:54:42 +0800 Subject: [PATCH 26/28] fix print error --- paddle/fluid/inference/analysis/analyzer_lac_tester.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/paddle/fluid/inference/analysis/analyzer_lac_tester.cc b/paddle/fluid/inference/analysis/analyzer_lac_tester.cc index b906b32cf5..522d870db8 100644 --- a/paddle/fluid/inference/analysis/analyzer_lac_tester.cc +++ b/paddle/fluid/inference/analysis/analyzer_lac_tester.cc @@ -182,6 +182,7 @@ void TestLACPrediction(const std::string &model_path, Timer timer; if (test_all_data) { double sum = 0; + LOG(INFO) << "Total number of samples: " << data.datasets.size(); for (int i = 0; i < repeat; i++) { for (size_t bid = 0; bid < data.batched_datas.size(); ++bid) { GetOneBatch(&input_slots, &data, batch_size); @@ -190,7 +191,9 @@ void TestLACPrediction(const std::string &model_path, sum += timer.toc(); } } - PrintTime(batch_size, repeat, 1, 0, sum / batch_size); + PrintTime(batch_size, repeat, 1, 0, sum / repeat); + LOG(INFO) << "Average latency of each sample: " + << sum / repeat / data.datasets.size() << " ms"; return; } timer.tic(); From f90c7865f0ababccd89332604e7e181c85cf9b60 Mon Sep 17 00:00:00 2001 From: Wu Yi Date: Sat, 8 Sep 2018 16:05:56 +0800 Subject: [PATCH 27/28] Benchmark tool for imgnet (#12305) * support test using executor without reader * run imgnet * update fluid benchmark * wip * update * update all models * support pyreader * update * clean up * make profile batches contollable * update API.spec * update scripts * clean dockerfile * update * clean comments * add scope argument docstring * use num_trainers to determine nccl init comms --- benchmark/fluid/Dockerfile | 4 +- benchmark/fluid/args.py | 16 +- benchmark/fluid/fluid_benchmark.py | 274 +++++++------- benchmark/fluid/imagenet_reader.py | 344 ++++++++++++++++++ benchmark/fluid/kube_gen_job.py | 13 + benchmark/fluid/models/__init__.py | 3 +- benchmark/fluid/models/machine_translation.py | 48 ++- benchmark/fluid/models/mnist.py | 95 +++-- benchmark/fluid/models/resnet.py | 197 ++++++---- .../fluid/models/resnet_with_preprocess.py | 268 ++++++++++++++ benchmark/fluid/models/se_resnext.py | 286 +++++++++++++++ .../fluid/models/stacked_dynamic_lstm.py | 75 ++-- benchmark/fluid/models/vgg.py | 101 +++-- paddle/fluid/API.spec | 2 +- paddle/fluid/platform/nccl_helper.h | 7 +- python/paddle/fluid/parallel_executor.py | 7 +- 16 files changed, 1342 insertions(+), 398 deletions(-) create mode 100644 benchmark/fluid/imagenet_reader.py create mode 100644 benchmark/fluid/models/resnet_with_preprocess.py create mode 100644 benchmark/fluid/models/se_resnext.py diff --git a/benchmark/fluid/Dockerfile b/benchmark/fluid/Dockerfile index 707fadb1fa..2e1e0d3768 100644 --- a/benchmark/fluid/Dockerfile +++ b/benchmark/fluid/Dockerfile @@ -11,6 +11,7 @@ RUN ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so.7 /usr/lib/libcudnn.so && ln -s # Add "ENV http_proxy=http://ip:port" if your download is slow, and don't forget to unset it at runtime. # exmaple: unset http_proxy && unset https_proxy && python fluid_benchmark.py ... + RUN pip install -U pip RUN pip install -U kubernetes paddlepaddle @@ -27,5 +28,6 @@ ADD *.whl / RUN pip install /*.whl && rm -f /*.whl ENV LD_LIBRARY_PATH=/usr/local/lib -ADD fluid_benchmark.py recordio_converter.py args.py recordio_converter.py run.sh run_fluid_benchmark.sh /workspace/ +ADD fluid_benchmark.py recordio_converter.py args.py recordio_converter.py run.sh run_fluid_benchmark.sh imagenet_reader.py /workspace/ ADD models/ /workspace/models/ + diff --git a/benchmark/fluid/args.py b/benchmark/fluid/args.py index a79f25ccc6..ed696e82f8 100644 --- a/benchmark/fluid/args.py +++ b/benchmark/fluid/args.py @@ -17,7 +17,8 @@ import argparse __all__ = ['parse_args', ] BENCHMARK_MODELS = [ - "machine_translation", "resnet", "vgg", "mnist", "stacked_dynamic_lstm" + "machine_translation", "resnet", "se_resnext", "vgg", "mnist", + "stacked_dynamic_lstm", "resnet_with_preprocess" ] @@ -67,12 +68,12 @@ def parse_args(): '--cpus', type=int, default=1, - help='If cpus > 1, will use ParallelDo to run, else use Executor.') + help='If cpus > 1, will set ParallelExecutor to use multiple threads.') parser.add_argument( '--data_set', type=str, default='flowers', - choices=['cifar10', 'flowers'], + choices=['cifar10', 'flowers', 'imagenet'], help='Optional dataset for benchmark.') parser.add_argument( '--infer_only', action='store_true', help='If set, run forward only.') @@ -122,6 +123,11 @@ def parse_args(): type=str, default="", help='Directory that contains all the training recordio files.') + parser.add_argument( + '--test_data_path', + type=str, + default="", + help='Directory that contains all the test data (NOT recordio).') parser.add_argument( '--use_inference_transpiler', action='store_true', @@ -130,5 +136,9 @@ def parse_args(): '--no_random', action='store_true', help='If set, keep the random seed and do not shuffle the data.') + parser.add_argument( + '--use_lars', + action='store_true', + help='If set, use lars for optimizers, ONLY support resnet module.') args = parser.parse_args() return args diff --git a/benchmark/fluid/fluid_benchmark.py b/benchmark/fluid/fluid_benchmark.py index 53d010434a..11bd75e1d0 100644 --- a/benchmark/fluid/fluid_benchmark.py +++ b/benchmark/fluid/fluid_benchmark.py @@ -16,6 +16,7 @@ import argparse import cProfile import time import os +import traceback import numpy as np @@ -27,7 +28,7 @@ import paddle.fluid.transpiler.distribute_transpiler as distribute_transpiler from args import * -def append_nccl2_prepare(trainer_id): +def append_nccl2_prepare(trainer_id, startup_prog): if trainer_id >= 0: # append gen_nccl_id at the end of startup program trainer_id = int(os.getenv("PADDLE_TRAINER_ID")) @@ -40,11 +41,11 @@ def append_nccl2_prepare(trainer_id): current_endpoint = os.getenv("PADDLE_CURRENT_IP") + ":" + port worker_endpoints.remove(current_endpoint) - nccl_id_var = fluid.default_startup_program().global_block().create_var( + nccl_id_var = startup_prog.global_block().create_var( name="NCCLID", persistable=True, type=fluid.core.VarDesc.VarType.RAW) - fluid.default_startup_program().global_block().append_op( + startup_prog.global_block().append_op( type="gen_nccl_id", inputs={}, outputs={"NCCLID": nccl_id_var}, @@ -59,7 +60,7 @@ def append_nccl2_prepare(trainer_id): "nccl-based dist train.") -def dist_transpile(trainer_id, args): +def dist_transpile(trainer_id, args, train_prog, startup_prog): if trainer_id < 0: return None, None @@ -80,133 +81,69 @@ def dist_transpile(trainer_id, args): # the role, should be either PSERVER or TRAINER training_role = os.getenv("PADDLE_TRAINING_ROLE") - t = distribute_transpiler.DistributeTranspiler() + config = distribute_transpiler.DistributeTranspilerConfig() + config.slice_var_up = not args.no_split_var + t = distribute_transpiler.DistributeTranspiler(config=config) t.transpile( trainer_id, + # NOTE: *MUST* use train_prog, for we are using with guard to + # generate different program for train and test. + program=train_prog, pservers=pserver_endpoints, trainers=trainers, sync_mode=not args.async_mode) if training_role == "PSERVER": pserver_program = t.get_pserver_program(current_endpoint) - pserver_startup_program = t.get_startup_program(current_endpoint, - pserver_program) + pserver_startup_program = t.get_startup_program( + current_endpoint, pserver_program, startup_program=startup_prog) return pserver_program, pserver_startup_program elif training_role == "TRAINER": train_program = t.get_trainer_program() - return train_program, fluid.default_startup_program() + return train_program, startup_prog else: raise ValueError( 'PADDLE_TRAINING_ROLE environment variable must be either TRAINER or PSERVER' ) -def test(exe, inference_program, test_reader, feeder, batch_acc): - accuracy_evaluator = fluid.metrics.Accuracy() - for batch_id, data in enumerate(test_reader()): - acc = exe.run(inference_program, - feed=feeder.feed(data), - fetch_list=[batch_acc]) - accuracy_evaluator.update(value=np.array(acc), weight=len(data)) +def test_parallel(exe, test_args, args, test_prog, feeder): + acc_evaluators = [] + for i in xrange(len(test_args[2])): + acc_evaluators.append(fluid.metrics.Accuracy()) - return accuracy_evaluator.eval() - - -# TODO(wuyi): replace train, train_parallel, test functions with new trainer -# API once it is ready. -def train(avg_loss, infer_prog, optimizer, train_reader, test_reader, batch_acc, - args, train_prog, startup_prog): - if os.getenv("PADDLE_TRAINING_ROLE") == "PSERVER": - place = core.CPUPlace() - exe = fluid.Executor(place) - exe.run(startup_prog) - exe.run(train_prog) - return - - if args.use_fake_data: - raise Exception( - "fake data is not supported in single GPU test for now.") - - place = core.CPUPlace() if args.device == 'CPU' else core.CUDAPlace(0) - exe = fluid.Executor(place) - exe.run(startup_prog) - - # Use inference_transpiler to speedup - if not args.use_reader_op: - feed_var_list = [ - var for var in train_prog.global_block().vars.itervalues() - if var.is_data - ] - feeder = fluid.DataFeeder(feed_var_list, place) - - iters, num_samples, start_time = 0, 0, time.time() - for pass_id in range(args.pass_num): - train_losses = [] - if not args.use_reader_op: - reader_generator = train_reader() - batch_id = 0 - data = None + to_fetch = [v.name for v in test_args[2]] + if args.use_reader_op: + test_args[4].start() while True: - if not args.use_reader_op: - data = next(reader_generator, None) - if data == None: - break - if iters == args.iterations: - reader_generator.close() + try: + acc_rets = exe.run(fetch_list=to_fetch) + for i, e in enumerate(acc_evaluators): + e.update( + value=np.array(acc_rets[i]), weight=args.batch_size) + except fluid.core.EOFException as eof: + test_args[4].reset() break - if iters == args.skip_batch_num: - start_time = time.time() - num_samples = 0 + else: + for batch_id, data in enumerate(test_args[3]()): + acc_rets = exe.run(feed=feeder.feed(data), fetch_list=to_fetch) + for i, e in enumerate(acc_evaluators): + e.update(value=np.array(acc_rets[i]), weight=len(data)) - if args.use_reader_op: - try: - loss = exe.run(train_prog, fetch_list=[avg_loss]) - except fluid.core.EnforceNotMet as ex: - break - else: - loss = exe.run(train_prog, - feed=feeder.feed(data), - fetch_list=[avg_loss]) - iters += 1 - batch_id += 1 - # FIXME(wuyi): For use_reader_op, if the current - # pass is not the last, the last batch of this pass - # is also equal to args.batch_size. - if args.use_reader_op: - num_samples += args.batch_size * args.gpus - else: - num_samples += len(data) - train_losses.append(loss) - print("Pass: %d, Iter: %d, Loss: %f\n" % - (pass_id, iters, np.mean(train_losses))) - print_train_time(start_time, time.time(), num_samples) - print("Pass: %d, Loss: %f" % (pass_id, np.mean(train_losses))), - # evaluation - if not args.no_test and batch_acc and not args.use_reader_op: - if args.use_inference_transpiler: - t = fluid.InferenceTranspiler() - t.transpile(infer_prog, place) - - pass_test_acc = test(exe, infer_prog, test_reader, feeder, - batch_acc) - print(", Test Accuracy: %f" % pass_test_acc) - print("\n") - # TODO(wuyi): add warmup passes to get better perf data. - exit(0) + return [e.eval() for e in acc_evaluators] -# TODO(wuyi): replace train, train_parallel, test functions with new trainer -# API once it is ready. -def train_parallel(avg_loss, infer_prog, optimizer, train_reader, test_reader, - batch_acc, args, train_prog, startup_prog, nccl_id_var, - num_trainers, trainer_id): +# NOTE: only need to benchmark using parallelexe +def train_parallel(train_args, test_args, args, train_prog, test_prog, + startup_prog, nccl_id_var, num_trainers, trainer_id): + over_all_start = time.time() place = core.CPUPlace() if args.device == 'CPU' else core.CUDAPlace(0) + feeder = None if not args.use_reader_op: feed_var_list = [ var for var in train_prog.global_block().vars.itervalues() if var.is_data ] feeder = fluid.DataFeeder(feed_var_list, place) - # generate fake: if args.use_fake_data: for var in feed_var_list: @@ -230,63 +167,110 @@ def train_parallel(avg_loss, infer_prog, optimizer, train_reader, test_reader, startup_exe = fluid.Executor(place) startup_exe.run(startup_prog) strategy = fluid.ExecutionStrategy() - strategy.num_threads = 1 + strategy.num_threads = args.cpus strategy.allow_op_delay = False + avg_loss = train_args[0] + + if args.update_method == "pserver": + # parameter server mode distributed training, merge + # gradients on local server, do not initialize + # ParallelExecutor with multi server all-reduce mode. + num_trainers = 1 + trainer_id = 0 + exe = fluid.ParallelExecutor( True, avg_loss.name, + main_program=train_prog, exec_strategy=strategy, num_trainers=num_trainers, trainer_id=trainer_id) + if not args.no_test: + if args.update_method == "pserver": + test_scope = None + else: + # NOTE: use an empty scope to avoid test exe using NCCLID + test_scope = fluid.Scope() + test_exe = fluid.ParallelExecutor( + True, main_program=test_prog, share_vars_from=exe) + for pass_id in range(args.pass_num): num_samples = 0 iters = 0 start_time = time.time() if not args.use_reader_op: - reader_generator = train_reader() + reader_generator = train_args[3]() #train_reader batch_id = 0 data = None + if args.use_reader_op: + train_args[4].start() while True: if not args.use_reader_op: data = next(reader_generator, None) if data == None: break + if args.profile and batch_id == 5: + profiler.start_profiler("All") + profiler.reset_profiler() + elif args.profile and batch_id == 10: + print("profiling total time: ", time.time() - start_time) + profiler.stop_profiler("total", "/tmp/profile_%d_pass%d" % + (trainer_id, pass_id)) if iters == args.iterations: reader_generator.close() break - if args.profile and pass_id == 0 and batch_id == 5: - profiler.start_profiler("All") - elif args.profile and pass_id == 0 and batch_id == 10: - profiler.stop_profiler("total", "/tmp/profile_%d" % trainer_id) if iters == args.skip_batch_num: start_time = time.time() num_samples = 0 + fetch_list = [avg_loss.name] + acc_name_list = [v.name for v in train_args[2]] + fetch_list.extend(acc_name_list) + if args.use_fake_data or args.use_reader_op: try: - loss, = exe.run([avg_loss.name]) + + fetch_ret = exe.run(fetch_list) + except fluid.core.EOFException as eof: + break except fluid.core.EnforceNotMet as ex: + traceback.print_exc() break else: - loss, = exe.run([avg_loss.name], feed=feeder.feed(data)) + fetch_ret = exe.run(fetch_list, feed=feeder.feed(data)) if args.use_reader_op: num_samples += args.batch_size * args.gpus else: num_samples += len(data) + iters += 1 if batch_id % 1 == 0: - print("Pass %d, batch %d, loss %s" % - (pass_id, batch_id, np.array(loss))) + fetched_data = [np.mean(np.array(d)) for d in fetch_ret] + print("Pass %d, batch %d, loss %s, accucacys: %s" % + (pass_id, batch_id, fetched_data[0], fetched_data[1:])) batch_id += 1 print_train_time(start_time, time.time(), num_samples) - if not args.no_test and batch_acc and not args.use_reader_op: - # we have not implement record io for test - # skip test when use args.use_reader_op - test_acc = test(startup_exe, infer_prog, test_reader, feeder, - batch_acc) - print("Pass: %d, Test Accuracy: %f\n" % (pass_id, test_acc)) + if args.use_reader_op: + train_args[4].reset() # reset reader handle + else: + del reader_generator + + if not args.no_test and test_args[2]: + test_feeder = None + if not args.use_reader_op: + test_feed_var_list = [ + var for var in test_prog.global_block().vars.itervalues() + if var.is_data + ] + test_feeder = fluid.DataFeeder(test_feed_var_list, place) + test_ret = test_parallel(test_exe, test_args, args, test_prog, + test_feeder) + print("Pass: %d, Test Accuracy: %s\n" % + (pass_id, [np.mean(np.array(v)) for v in test_ret])) + + print("total train time: ", time.time() - over_all_start) def print_arguments(args): @@ -328,44 +312,46 @@ def main(): if args.use_cprof: pr = cProfile.Profile() pr.enable() + model_def = __import__("models.%s" % args.model, fromlist=["models"]) - train_args = list(model_def.get_model(args)) - train_args.append(args) - # Run optimizer.minimize(avg_loss) - train_args[2].minimize(train_args[0]) - if args.memory_optimize: - fluid.memory_optimize(fluid.default_main_program()) + + train_prog = fluid.Program() + test_prog = fluid.Program() + startup_prog = fluid.Program() + + train_args = list(model_def.get_model(args, True, train_prog, startup_prog)) + test_args = list(model_def.get_model(args, False, test_prog, startup_prog)) + + all_args = [train_args, test_args, args] if args.update_method == "pserver": - train_prog, startup_prog = dist_transpile(trainer_id, args) + train_prog, startup_prog = dist_transpile(trainer_id, args, train_prog, + startup_prog) if not train_prog: raise Exception( "Must configure correct environments to run dist train.") - train_args.extend([train_prog, startup_prog]) + all_args.extend([train_prog, test_prog, startup_prog]) if args.gpus > 1 and os.getenv("PADDLE_TRAINING_ROLE") == "TRAINER": - train_args.extend([nccl_id_var, num_trainers, trainer_id]) - train_parallel(*train_args) - train(*train_args) + all_args.extend([nccl_id_var, num_trainers, trainer_id]) + train_parallel(*all_args) + elif os.getenv("PADDLE_TRAINING_ROLE") == "PSERVER": + # start pserver with Executor + server_exe = fluid.Executor(fluid.CPUPlace()) + server_exe.run(startup_prog) + server_exe.run(train_prog) exit(0) # for other update methods, use default programs - train_args.append(fluid.default_main_program()) - train_args.append(fluid.default_startup_program()) + all_args.extend([train_prog, test_prog, startup_prog]) if args.update_method == "nccl2": - nccl_id_var, num_trainers, trainer_id = append_nccl2_prepare(trainer_id) - if args.gpus == 1: - # NOTE: parallel executor use profiler interanlly - if args.use_nvprof and args.device == 'GPU': - with profiler.cuda_profiler("cuda_profiler.txt", 'csv') as nvprof: - train(*train_args) - else: - train(*train_args) - else: - if args.device == "CPU": - raise Exception("Only support GPU perf with parallel exe") - train_args.extend([nccl_id_var, num_trainers, trainer_id]) - train_parallel(*train_args) + nccl_id_var, num_trainers, trainer_id = append_nccl2_prepare( + trainer_id, startup_prog) + + if args.device == "CPU": + raise Exception("Only support GPU perf with parallel exe") + all_args.extend([nccl_id_var, num_trainers, trainer_id]) + train_parallel(*all_args) if __name__ == "__main__": diff --git a/benchmark/fluid/imagenet_reader.py b/benchmark/fluid/imagenet_reader.py new file mode 100644 index 0000000000..a39485a61f --- /dev/null +++ b/benchmark/fluid/imagenet_reader.py @@ -0,0 +1,344 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import math +import random +import functools +import numpy as np +from threading import Thread +import subprocess +import time + +from Queue import Queue +import paddle +from PIL import Image, ImageEnhance + +random.seed(0) + +DATA_DIM = 224 + +THREAD = int(os.getenv("PREPROCESS_THREADS", "10")) +BUF_SIZE = 5120 + +DATA_DIR = '/mnt/ImageNet' +TRAIN_LIST = '/mnt/ImageNet/train.txt' +TEST_LIST = '/mnt/ImageNet/val.txt' + +img_mean = np.array([0.485, 0.456, 0.406]).reshape((3, 1, 1)) +img_std = np.array([0.229, 0.224, 0.225]).reshape((3, 1, 1)) + + +def resize_short(img, target_size): + percent = float(target_size) / min(img.size[0], img.size[1]) + resized_width = int(round(img.size[0] * percent)) + resized_height = int(round(img.size[1] * percent)) + img = img.resize((resized_width, resized_height), Image.LANCZOS) + return img + + +def crop_image(img, target_size, center): + width, height = img.size + size = target_size + if center == True: + w_start = (width - size) / 2 + h_start = (height - size) / 2 + else: + w_start = random.randint(0, width - size) + h_start = random.randint(0, height - size) + w_end = w_start + size + h_end = h_start + size + img = img.crop((w_start, h_start, w_end, h_end)) + return img + + +def random_crop(img, size, scale=[0.08, 1.0], ratio=[3. / 4., 4. / 3.]): + aspect_ratio = math.sqrt(random.uniform(*ratio)) + w = 1. * aspect_ratio + h = 1. / aspect_ratio + + bound = min((float(img.size[0]) / img.size[1]) / (w**2), + (float(img.size[1]) / img.size[0]) / (h**2)) + scale_max = min(scale[1], bound) + scale_min = min(scale[0], bound) + + target_area = img.size[0] * img.size[1] * random.uniform(scale_min, + scale_max) + target_size = math.sqrt(target_area) + w = int(target_size * w) + h = int(target_size * h) + + i = random.randint(0, img.size[0] - w) + j = random.randint(0, img.size[1] - h) + + img = img.crop((i, j, i + w, j + h)) + img = img.resize((size, size), Image.LANCZOS) + return img + + +def rotate_image(img): + angle = random.randint(-10, 10) + img = img.rotate(angle) + return img + + +def distort_color(img): + def random_brightness(img, lower=0.5, upper=1.5): + e = random.uniform(lower, upper) + return ImageEnhance.Brightness(img).enhance(e) + + def random_contrast(img, lower=0.5, upper=1.5): + e = random.uniform(lower, upper) + return ImageEnhance.Contrast(img).enhance(e) + + def random_color(img, lower=0.5, upper=1.5): + e = random.uniform(lower, upper) + return ImageEnhance.Color(img).enhance(e) + + ops = [random_brightness, random_contrast, random_color] + random.shuffle(ops) + + img = ops[0](img) + img = ops[1](img) + img = ops[2](img) + + return img + + +def process_image(sample, mode, color_jitter, rotate): + img_path = sample[0] + + img = Image.open(img_path) + if mode == 'train': + if rotate: img = rotate_image(img) + img = random_crop(img, DATA_DIM) + else: + img = resize_short(img, target_size=256) + img = crop_image(img, target_size=DATA_DIM, center=True) + if mode == 'train': + if color_jitter: + img = distort_color(img) + if random.randint(0, 1) == 1: + img = img.transpose(Image.FLIP_LEFT_RIGHT) + + if img.mode != 'RGB': + img = img.convert('RGB') + + img = np.array(img).astype('float32').transpose((2, 0, 1)) / 255 + img -= img_mean + img /= img_std + + if mode == 'train' or mode == 'val': + return img, sample[1] + elif mode == 'test': + return [img] + + +class XmapEndSignal(): + pass + + +def xmap_readers(mapper, + reader, + process_num, + buffer_size, + order=False, + print_queue_state=True): + end = XmapEndSignal() + + # define a worker to read samples from reader to in_queue + def read_worker(reader, in_queue): + for i in reader(): + in_queue.put(i) + in_queue.put(end) + + # define a worker to read samples from reader to in_queue with order flag + def order_read_worker(reader, in_queue, file_queue): + in_order = 0 + for i in reader(): + in_queue.put((in_order, i)) + in_order += 1 + in_queue.put(end) + + # define a worker to handle samples from in_queue by mapper + # and put mapped samples into out_queue + def handle_worker(in_queue, out_queue, mapper): + sample = in_queue.get() + while not isinstance(sample, XmapEndSignal): + r = mapper(sample) + out_queue.put(r) + sample = in_queue.get() + in_queue.put(end) + out_queue.put(end) + + # define a worker to handle samples from in_queue by mapper + # and put mapped samples into out_queue by order + def order_handle_worker(in_queue, out_queue, mapper, out_order): + ins = in_queue.get() + while not isinstance(ins, XmapEndSignal): + order, sample = ins + r = mapper(sample) + while order != out_order[0]: + pass + out_queue.put(r) + out_order[0] += 1 + ins = in_queue.get() + in_queue.put(end) + out_queue.put(end) + + def xreader(): + file_queue = Queue() + in_queue = Queue(buffer_size) + out_queue = Queue(buffer_size) + out_order = [0] + # start a read worker in a thread + target = order_read_worker if order else read_worker + t = Thread(target=target, args=(reader, in_queue)) + t.daemon = True + t.start() + # start several handle_workers + target = order_handle_worker if order else handle_worker + args = (in_queue, out_queue, mapper, out_order) if order else ( + in_queue, out_queue, mapper) + workers = [] + for i in xrange(process_num): + worker = Thread(target=target, args=args) + worker.daemon = True + workers.append(worker) + for w in workers: + w.start() + + sample = out_queue.get() + start_t = time.time() + while not isinstance(sample, XmapEndSignal): + yield sample + sample = out_queue.get() + if time.time() - start_t > 3: + if print_queue_state: + print("queue sizes: ", in_queue.qsize(), out_queue.qsize()) + start_t = time.time() + finish = 1 + while finish < process_num: + sample = out_queue.get() + if isinstance(sample, XmapEndSignal): + finish += 1 + else: + yield sample + + return xreader + + +def _reader_creator(file_list, + mode, + shuffle=False, + color_jitter=False, + rotate=False, + xmap=True): + def reader(): + with open(file_list) as flist: + full_lines = [line.strip() for line in flist] + if shuffle: + random.shuffle(full_lines) + if mode == 'train': + trainer_id = int(os.getenv("PADDLE_TRAINER_ID")) + trainer_count = int(os.getenv("PADDLE_TRAINERS")) + per_node_lines = len(full_lines) / trainer_count + lines = full_lines[trainer_id * per_node_lines:(trainer_id + 1) + * per_node_lines] + print( + "read images from %d, length: %d, lines length: %d, total: %d" + % (trainer_id * per_node_lines, per_node_lines, len(lines), + len(full_lines))) + else: + lines = full_lines + + for line in lines: + if mode == 'train': + img_path, label = line.split() + img_path = img_path.replace("JPEG", "jpeg") + img_path = os.path.join(DATA_DIR, "train", img_path) + yield (img_path, int(label)) + elif mode == 'val': + img_path, label = line.split() + img_path = img_path.replace("JPEG", "jpeg") + img_path = os.path.join(DATA_DIR, "val", img_path) + yield (img_path, int(label)) + elif mode == 'test': + img_path = os.path.join(DATA_DIR, line) + yield [img_path] + + mapper = functools.partial( + process_image, mode=mode, color_jitter=color_jitter, rotate=rotate) + + return paddle.reader.xmap_readers(mapper, reader, THREAD, BUF_SIZE) + + +def load_raw_image_uint8(sample): + img_arr = np.array(Image.open(sample[0])).astype('int64') + return img_arr, int(sample[1]) + + +def train_raw(file_list=TRAIN_LIST, shuffle=True): + def reader(): + with open(file_list) as flist: + full_lines = [line.strip() for line in flist] + if shuffle: + random.shuffle(full_lines) + + trainer_id = int(os.getenv("PADDLE_TRAINER_ID")) + trainer_count = int(os.getenv("PADDLE_TRAINERS")) + per_node_lines = len(full_lines) / trainer_count + lines = full_lines[trainer_id * per_node_lines:(trainer_id + 1) * + per_node_lines] + print("read images from %d, length: %d, lines length: %d, total: %d" + % (trainer_id * per_node_lines, per_node_lines, len(lines), + len(full_lines))) + + for line in lines: + img_path, label = line.split() + img_path = img_path.replace("JPEG", "jpeg") + img_path = os.path.join(DATA_DIR, "train", img_path) + yield (img_path, int(label)) + + return paddle.reader.xmap_readers(load_raw_image_uint8, reader, THREAD, + BUF_SIZE) + + +def train(file_list=TRAIN_LIST, xmap=True): + return _reader_creator( + file_list, + 'train', + shuffle=True, + color_jitter=False, + rotate=False, + xmap=xmap) + + +def val(file_list=TEST_LIST, xmap=True): + return _reader_creator(file_list, 'val', shuffle=False, xmap=xmap) + + +def test(file_list=TEST_LIST): + return _reader_creator(file_list, 'test', shuffle=False) + + +if __name__ == "__main__": + c = 0 + start_t = time.time() + for d in train()(): + c += 1 + if c >= 10000: + break + spent = time.time() - start_t + print("read 10000 speed: ", 10000 / spent, spent) diff --git a/benchmark/fluid/kube_gen_job.py b/benchmark/fluid/kube_gen_job.py index dfe8b5cdd5..c1f22f1bfa 100644 --- a/benchmark/fluid/kube_gen_job.py +++ b/benchmark/fluid/kube_gen_job.py @@ -163,6 +163,19 @@ def gen_job(): volumes.append({"name": "dshm", "emptyDir": {"medium": "Memory"}}) volumeMounts.append({"mountPath": "/dev/shm", "name": "dshm"}) + # add ceph volumes + volumes.append({ + "name": "ceph-data", + "cephfs": { + "monitors": ["192.168.16.23:6789"], + "secretRef": { + "name": "ceph-secret" + }, + "user": "admin", + } + }) + volumeMounts.append({"mountPath": "/mnt/data", "name": "ceph-data"}) + tn["spec"]["template"]["spec"]["volumes"] = volumes tn_container["volumeMounts"] = volumeMounts diff --git a/benchmark/fluid/models/__init__.py b/benchmark/fluid/models/__init__.py index 1c3fcac8dd..1b8f63c707 100644 --- a/benchmark/fluid/models/__init__.py +++ b/benchmark/fluid/models/__init__.py @@ -13,5 +13,6 @@ # limitations under the License. __all__ = [ - "machine_translation", "resnet", "vgg", "mnist", "stacked_dynamic_lstm" + "machine_translation", "resnet", "vgg", "mnist", "stacked_dynamic_lstm", + "resnet_with_preprocess" ] diff --git a/benchmark/fluid/models/machine_translation.py b/benchmark/fluid/models/machine_translation.py index 17f6b03826..18163c35d6 100644 --- a/benchmark/fluid/models/machine_translation.py +++ b/benchmark/fluid/models/machine_translation.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """seq2seq model for fluid.""" + from __future__ import absolute_import from __future__ import division from __future__ import print_function @@ -181,7 +182,7 @@ def lodtensor_to_ndarray(lod_tensor): return ndarray -def get_model(args): +def get_model(args, is_train, main_prog, startup_prog): if args.use_reader_op: raise Exception("machine_translation do not support reader op for now.") embedding_dim = 512 @@ -190,30 +191,27 @@ def get_model(args): dict_size = 30000 beam_size = 3 max_length = 250 - avg_cost, feeding_list = seq_to_seq_net( - embedding_dim, - encoder_size, - decoder_size, - dict_size, - dict_size, - False, - beam_size=beam_size, - max_length=max_length) - - # clone from default main program - inference_program = fluid.default_main_program().clone() - - optimizer = fluid.optimizer.Adam(learning_rate=args.learning_rate) - - train_batch_generator = paddle.batch( - paddle.reader.shuffle( - paddle.dataset.wmt14.train(dict_size), buf_size=1000), - batch_size=args.batch_size * args.gpus) - test_batch_generator = paddle.batch( + with fluid.program_guard(main_prog, startup_prog): + with fluid.unique_name.guard(): + avg_cost, feeding_list = seq_to_seq_net( + embedding_dim, + encoder_size, + decoder_size, + dict_size, + dict_size, + False, + beam_size=beam_size, + max_length=max_length) + if is_train: + optimizer = fluid.optimizer.Adam(learning_rate=args.learning_rate) + optimizer.minimize(avg_cost) + + batch_generator = paddle.batch( paddle.reader.shuffle( - paddle.dataset.wmt14.test(dict_size), buf_size=1000), - batch_size=args.batch_size) + paddle.dataset.wmt14.train(dict_size) + if is_train else paddle.dataset.wmt14.test(dict_size), + buf_size=1000), + batch_size=args.batch_size * args.gpus) - return avg_cost, inference_program, optimizer, train_batch_generator, \ - test_batch_generator, None + return avg_cost, optimizer, [], batch_generator, None diff --git a/benchmark/fluid/models/mnist.py b/benchmark/fluid/models/mnist.py index 8e740dc689..cef8657ee6 100644 --- a/benchmark/fluid/models/mnist.py +++ b/benchmark/fluid/models/mnist.py @@ -65,61 +65,50 @@ def cnn_model(data): return predict -def get_model(args): - if args.use_reader_op: - filelist = [ - os.path.join(args.data_path, f) for f in os.listdir(args.data_path) - ] - data_file = fluid.layers.open_files( - filenames=filelist, - shapes=[[-1, 1, 28, 28], (-1, 1)], - lod_levels=[0, 0], - dtypes=["float32", "int64"], - thread_num=args.gpus, - pass_num=args.pass_num) - data_file = fluid.layers.double_buffer( - fluid.layers.batch( - data_file, batch_size=args.batch_size)) - images, label = fluid.layers.read_file(data_file) - else: - images = fluid.layers.data(name='pixel', shape=[1, 28, 28], dtype=DTYPE) - label = fluid.layers.data(name='label', shape=[1], dtype='int64') - - if args.device == 'CPU' and args.cpus > 1: - places = fluid.layers.get_places(args.cpus) - pd = fluid.layers.ParallelDo(places) - with pd.do(): - predict = cnn_model(pd.read_input(images)) - label = pd.read_input(label) +def get_model(args, is_train, main_prog, startup_prog): + # NOTE: mnist is small, we don't implement data sharding yet. + filelist = [ + os.path.join(args.data_path, f) for f in os.listdir(args.data_path) + ] + with fluid.program_guard(main_prog, startup_prog): + if args.use_reader_op: + data_file_handle = fluid.layers.open_files( + filenames=filelist, + shapes=[[-1, 1, 28, 28], (-1, 1)], + lod_levels=[0, 0], + dtypes=["float32", "int64"], + thread_num=1, + pass_num=1) + data_file = fluid.layers.double_buffer( + fluid.layers.batch( + data_file_handle, batch_size=args.batch_size)) + with fluid.unique_name.guard(): + if args.use_reader_op: + input, label = fluid.layers.read_file(data_file) + else: + images = fluid.layers.data( + name='pixel', shape=[1, 28, 28], dtype='float32') + label = fluid.layers.data( + name='label', shape=[1], dtype='int64') + + predict = cnn_model(images) cost = fluid.layers.cross_entropy(input=predict, label=label) avg_cost = fluid.layers.mean(x=cost) + # Evaluator batch_acc = fluid.layers.accuracy(input=predict, label=label) - - pd.write_output(avg_cost) - pd.write_output(batch_acc) - - avg_cost, batch_acc = pd() - avg_cost = fluid.layers.mean(avg_cost) - batch_acc = fluid.layers.mean(batch_acc) - else: - # Train program - predict = cnn_model(images) - cost = fluid.layers.cross_entropy(input=predict, label=label) - avg_cost = fluid.layers.mean(x=cost) - - # Evaluator - batch_acc = fluid.layers.accuracy(input=predict, label=label) - - # inference program - inference_program = fluid.default_main_program().clone() - - # Optimization - opt = fluid.optimizer.AdamOptimizer( - learning_rate=0.001, beta1=0.9, beta2=0.999) + # Optimization + if is_train: + opt = fluid.optimizer.AdamOptimizer( + learning_rate=0.001, beta1=0.9, beta2=0.999) + opt.minimize() + if args.memory_optimize: + fluid.memory_optimize(main_prog) # Reader - train_reader = paddle.batch( - paddle.dataset.mnist.train(), batch_size=args.batch_size * args.gpus) - test_reader = paddle.batch( - paddle.dataset.mnist.test(), batch_size=args.batch_size) - return avg_cost, inference_program, opt, train_reader, test_reader, batch_acc + if is_train: + reader = paddle.dataset.mnist.train() + else: + reader = paddle.dataset.mnist.test() + batched_reader = paddle.batch( + reader, batch_size=args.batch_size * args.gpus) + return avg_cost, opt, [batch_acc], batched_reader, data_file_handle diff --git a/benchmark/fluid/models/resnet.py b/benchmark/fluid/models/resnet.py index d44a9c07d3..ae1baa48e1 100644 --- a/benchmark/fluid/models/resnet.py +++ b/benchmark/fluid/models/resnet.py @@ -27,10 +27,17 @@ import paddle import paddle.fluid as fluid import paddle.fluid.core as core import paddle.fluid.profiler as profiler -from recordio_converter import imagenet_train, imagenet_test +# from recordio_converter import imagenet_train, imagenet_test +from imagenet_reader import train, val -def conv_bn_layer(input, ch_out, filter_size, stride, padding, act='relu'): +def conv_bn_layer(input, + ch_out, + filter_size, + stride, + padding, + act='relu', + is_train=True): conv1 = fluid.layers.conv2d( input=input, filter_size=filter_size, @@ -39,29 +46,31 @@ def conv_bn_layer(input, ch_out, filter_size, stride, padding, act='relu'): padding=padding, act=None, bias_attr=False) - return fluid.layers.batch_norm(input=conv1, act=act) + return fluid.layers.batch_norm(input=conv1, act=act, is_test=not is_train) -def shortcut(input, ch_out, stride): +def shortcut(input, ch_out, stride, is_train=True): ch_in = input.shape[1] # if args.data_format == 'NCHW' else input.shape[-1] if ch_in != ch_out: - return conv_bn_layer(input, ch_out, 1, stride, 0, None) + return conv_bn_layer( + input, ch_out, 1, stride, 0, None, is_train=is_train) else: return input -def basicblock(input, ch_out, stride): - short = shortcut(input, ch_out, stride) - conv1 = conv_bn_layer(input, ch_out, 3, stride, 1) - conv2 = conv_bn_layer(conv1, ch_out, 3, 1, 1, act=None) +def basicblock(input, ch_out, stride, is_train=True): + short = shortcut(input, ch_out, stride, is_train=is_train) + conv1 = conv_bn_layer(input, ch_out, 3, stride, 1, is_train=is_train) + conv2 = conv_bn_layer(conv1, ch_out, 3, 1, 1, act=None, is_train=is_train) return fluid.layers.elementwise_add(x=short, y=conv2, act='relu') -def bottleneck(input, ch_out, stride): - short = shortcut(input, ch_out * 4, stride) - conv1 = conv_bn_layer(input, ch_out, 1, stride, 0) - conv2 = conv_bn_layer(conv1, ch_out, 3, 1, 1) - conv3 = conv_bn_layer(conv2, ch_out * 4, 1, 1, 0, act=None) +def bottleneck(input, ch_out, stride, is_train=True): + short = shortcut(input, ch_out * 4, stride, is_train=is_train) + conv1 = conv_bn_layer(input, ch_out, 1, stride, 0, is_train=is_train) + conv2 = conv_bn_layer(conv1, ch_out, 3, 1, 1, is_train=is_train) + conv3 = conv_bn_layer( + conv2, ch_out * 4, 1, 1, 0, act=None, is_train=is_train) return fluid.layers.elementwise_add(x=short, y=conv3, act='relu') @@ -72,7 +81,11 @@ def layer_warp(block_func, input, ch_out, count, stride): return res_out -def resnet_imagenet(input, class_dim, depth=50, data_format='NCHW'): +def resnet_imagenet(input, + class_dim, + depth=50, + data_format='NCHW', + is_train=True): cfg = { 18: ([2, 2, 2, 1], basicblock), @@ -115,8 +128,9 @@ def resnet_cifar10(input, class_dim, depth=32, data_format='NCHW'): return out -def get_model(args): +def _model_reader_dshape_classdim(args, is_train): model = resnet_cifar10 + reader = None if args.data_set == "cifar10": class_dim = 10 if args.data_format == 'NCHW': @@ -124,8 +138,10 @@ def get_model(args): else: dshape = [32, 32, 3] model = resnet_cifar10 - train_reader = paddle.dataset.cifar.train10() - test_reader = paddle.dataset.cifar.test10() + if is_train: + reader = paddle.dataset.cifar.train10() + else: + reader = paddle.dataset.cifar.test10() elif args.data_set == "flowers": class_dim = 102 if args.data_format == 'NCHW': @@ -133,8 +149,10 @@ def get_model(args): else: dshape = [224, 224, 3] model = resnet_imagenet - train_reader = paddle.dataset.flowers.train() - test_reader = paddle.dataset.flowers.test() + if is_train: + reader = paddle.dataset.flowers.train() + else: + reader = paddle.dataset.flowers.test() elif args.data_set == "imagenet": class_dim = 1000 if args.data_format == 'NCHW': @@ -145,64 +163,89 @@ def get_model(args): if not args.data_path: raise Exception( "Must specify --data_path when training with imagenet") - train_reader = imagenet_train(args.data_path) - test_reader = imagenet_test(args.data_path) - - if args.use_reader_op: - filelist = [ - os.path.join(args.data_path, f) for f in os.listdir(args.data_path) - ] - data_file = fluid.layers.open_files( - filenames=filelist, - shapes=[[-1] + dshape, (-1, 1)], - lod_levels=[0, 0], - dtypes=["float32", "int64"], - thread_num=args.gpus, - pass_num=args.pass_num) - data_file = fluid.layers.double_buffer( - fluid.layers.batch( - data_file, batch_size=args.batch_size)) - input, label = fluid.layers.read_file(data_file) - else: - input = fluid.layers.data(name='data', shape=dshape, dtype='float32') - label = fluid.layers.data(name='label', shape=[1], dtype='int64') - - if args.device == 'CPU' and args.cpus > 1: - places = fluid.layers.get_places(args.cpus) - pd = fluid.layers.ParallelDo(places) - with pd.do(): - predict = model(pd.read_input(input), class_dim) - label = pd.read_input(label) + if not args.use_reader_op: + if is_train: + reader = train() + else: + reader = val() + else: + if is_train: + reader = train(xmap=False) + else: + reader = val(xmap=False) + return model, reader, dshape, class_dim + + +def get_model(args, is_train, main_prog, startup_prog): + model, reader, dshape, class_dim = _model_reader_dshape_classdim(args, + is_train) + + pyreader = None + trainer_count = int(os.getenv("PADDLE_TRAINERS")) + with fluid.program_guard(main_prog, startup_prog): + with fluid.unique_name.guard(): + if args.use_reader_op: + pyreader = fluid.layers.py_reader( + capacity=args.batch_size * args.gpus, + shapes=([-1] + dshape, (-1, 1)), + dtypes=('float32', 'int64'), + name="train_reader" if is_train else "test_reader", + use_double_buffer=True) + input, label = fluid.layers.read_file(pyreader) + else: + input = fluid.layers.data( + name='data', shape=dshape, dtype='float32') + label = fluid.layers.data( + name='label', shape=[1], dtype='int64') + + predict = model(input, class_dim, is_train=is_train) cost = fluid.layers.cross_entropy(input=predict, label=label) avg_cost = fluid.layers.mean(x=cost) - batch_acc = fluid.layers.accuracy(input=predict, label=label) - - pd.write_output(avg_cost) - pd.write_output(batch_acc) - avg_cost, batch_acc = pd() - avg_cost = fluid.layers.mean(avg_cost) - batch_acc = fluid.layers.mean(batch_acc) + batch_acc1 = fluid.layers.accuracy(input=predict, label=label, k=1) + batch_acc5 = fluid.layers.accuracy(input=predict, label=label, k=5) + + # configure optimize + optimizer = None + if is_train: + if args.use_lars: + lars_decay = 1.0 + else: + lars_decay = 0.0 + + total_images = 1281167 / trainer_count + + step = int(total_images / args.batch_size + 1) + epochs = [30, 60, 80, 90] + bd = [step * e for e in epochs] + base_lr = args.learning_rate + lr = [] + lr = [base_lr * (0.1**i) for i in range(len(bd) + 1)] + optimizer = fluid.optimizer.Momentum( + learning_rate=base_lr, + #learning_rate=fluid.layers.piecewise_decay( + # boundaries=bd, values=lr), + momentum=0.9, + regularization=fluid.regularizer.L2Decay(1e-4)) + optimizer.minimize(avg_cost) + + if args.memory_optimize: + fluid.memory_optimize(main_prog) + + # config readers + if not args.use_reader_op: + batched_reader = paddle.batch( + reader if args.no_random else paddle.reader.shuffle( + reader, buf_size=5120), + batch_size=args.batch_size * args.gpus, + drop_last=True) else: - predict = model(input, class_dim) - cost = fluid.layers.cross_entropy(input=predict, label=label) - avg_cost = fluid.layers.mean(x=cost) - batch_acc = fluid.layers.accuracy(input=predict, label=label) - - inference_program = fluid.default_main_program().clone() - with fluid.program_guard(inference_program): - inference_program = fluid.io.get_inference_program( - target_vars=[batch_acc]) - - optimizer = fluid.optimizer.Momentum(learning_rate=0.01, momentum=0.9) - - batched_train_reader = paddle.batch( - train_reader if args.no_random else paddle.reader.shuffle( - train_reader, buf_size=5120), - batch_size=args.batch_size * args.gpus, - drop_last=True) - batched_test_reader = paddle.batch( - test_reader, batch_size=args.batch_size, drop_last=True) - - return avg_cost, inference_program, optimizer, batched_train_reader,\ - batched_test_reader, batch_acc + batched_reader = None + pyreader.decorate_paddle_reader( + paddle.batch( + reader if args.no_random else paddle.reader.shuffle( + reader, buf_size=5120), + batch_size=args.batch_size)) + + return avg_cost, optimizer, [batch_acc1, + batch_acc5], batched_reader, pyreader diff --git a/benchmark/fluid/models/resnet_with_preprocess.py b/benchmark/fluid/models/resnet_with_preprocess.py new file mode 100644 index 0000000000..e8d661d847 --- /dev/null +++ b/benchmark/fluid/models/resnet_with_preprocess.py @@ -0,0 +1,268 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import functools +import numpy as np +import time +import os + +import cProfile, pstats, StringIO + +import paddle +import paddle.fluid as fluid +import paddle.fluid.core as core +import paddle.fluid.profiler as profiler +# from recordio_converter import imagenet_train, imagenet_test +from imagenet_reader import train_raw, val + + +def conv_bn_layer(input, + ch_out, + filter_size, + stride, + padding, + act='relu', + is_train=True): + conv1 = fluid.layers.conv2d( + input=input, + filter_size=filter_size, + num_filters=ch_out, + stride=stride, + padding=padding, + act=None, + bias_attr=False) + return fluid.layers.batch_norm(input=conv1, act=act, is_test=not is_train) + + +def shortcut(input, ch_out, stride, is_train=True): + ch_in = input.shape[1] # if args.data_format == 'NCHW' else input.shape[-1] + if ch_in != ch_out: + return conv_bn_layer( + input, ch_out, 1, stride, 0, None, is_train=is_train) + else: + return input + + +def basicblock(input, ch_out, stride, is_train=True): + short = shortcut(input, ch_out, stride, is_train=is_train) + conv1 = conv_bn_layer(input, ch_out, 3, stride, 1, is_train=is_train) + conv2 = conv_bn_layer(conv1, ch_out, 3, 1, 1, act=None, is_train=is_train) + return fluid.layers.elementwise_add(x=short, y=conv2, act='relu') + + +def bottleneck(input, ch_out, stride, is_train=True): + short = shortcut(input, ch_out * 4, stride, is_train=is_train) + conv1 = conv_bn_layer(input, ch_out, 1, stride, 0, is_train=is_train) + conv2 = conv_bn_layer(conv1, ch_out, 3, 1, 1, is_train=is_train) + conv3 = conv_bn_layer( + conv2, ch_out * 4, 1, 1, 0, act=None, is_train=is_train) + return fluid.layers.elementwise_add(x=short, y=conv3, act='relu') + + +def layer_warp(block_func, input, ch_out, count, stride): + res_out = block_func(input, ch_out, stride) + for i in range(1, count): + res_out = block_func(res_out, ch_out, 1) + return res_out + + +def resnet_imagenet(input, + class_dim, + depth=50, + data_format='NCHW', + is_train=True): + + cfg = { + 18: ([2, 2, 2, 1], basicblock), + 34: ([3, 4, 6, 3], basicblock), + 50: ([3, 4, 6, 3], bottleneck), + 101: ([3, 4, 23, 3], bottleneck), + 152: ([3, 8, 36, 3], bottleneck) + } + stages, block_func = cfg[depth] + conv1 = conv_bn_layer(input, ch_out=64, filter_size=7, stride=2, padding=3) + pool1 = fluid.layers.pool2d( + input=conv1, pool_type='avg', pool_size=3, pool_stride=2) + res1 = layer_warp(block_func, pool1, 64, stages[0], 1) + res2 = layer_warp(block_func, res1, 128, stages[1], 2) + res3 = layer_warp(block_func, res2, 256, stages[2], 2) + res4 = layer_warp(block_func, res3, 512, stages[3], 2) + pool2 = fluid.layers.pool2d( + input=res4, + pool_size=7, + pool_type='avg', + pool_stride=1, + global_pooling=True) + out = fluid.layers.fc(input=pool2, size=class_dim, act='softmax') + return out + + +def resnet_cifar10(input, class_dim, depth=32, data_format='NCHW'): + assert (depth - 2) % 6 == 0 + + n = (depth - 2) // 6 + + conv1 = conv_bn_layer( + input=input, ch_out=16, filter_size=3, stride=1, padding=1) + res1 = layer_warp(basicblock, conv1, 16, n, 1) + res2 = layer_warp(basicblock, res1, 32, n, 2) + res3 = layer_warp(basicblock, res2, 64, n, 2) + pool = fluid.layers.pool2d( + input=res3, pool_size=8, pool_type='avg', pool_stride=1) + out = fluid.layers.fc(input=pool, size=class_dim, act='softmax') + return out + + +def _model_reader_dshape_classdim(args, is_train): + model = resnet_cifar10 + reader = None + if args.data_set == "cifar10": + class_dim = 10 + if args.data_format == 'NCHW': + dshape = [3, 32, 32] + else: + dshape = [32, 32, 3] + model = resnet_cifar10 + if is_train: + reader = paddle.dataset.cifar.train10() + else: + reader = paddle.dataset.cifar.test10() + elif args.data_set == "flowers": + class_dim = 102 + if args.data_format == 'NCHW': + dshape = [3, 224, 224] + else: + dshape = [224, 224, 3] + model = resnet_imagenet + if is_train: + reader = paddle.dataset.flowers.train() + else: + reader = paddle.dataset.flowers.test() + elif args.data_set == "imagenet": + class_dim = 1000 + if args.data_format == 'NCHW': + dshape = [3, 224, 224] + else: + dshape = [224, 224, 3] + model = resnet_imagenet + if not args.data_path: + raise Exception( + "Must specify --data_path when training with imagenet") + if not args.use_reader_op: + if is_train: + reader = train_raw() + else: + reader = val() + else: + if is_train: + reader = train_raw() + else: + reader = val(xmap=False) + return model, reader, dshape, class_dim + + +def get_model(args, is_train, main_prog, startup_prog): + model, reader, dshape, class_dim = _model_reader_dshape_classdim(args, + is_train) + + pyreader = None + trainer_count = int(os.getenv("PADDLE_TRAINERS")) + with fluid.program_guard(main_prog, startup_prog): + with fluid.unique_name.guard(): + if args.use_reader_op: + pyreader = fluid.layers.py_reader( + capacity=args.batch_size * args.gpus, + shapes=([-1] + dshape, (-1, 1)), + dtypes=('uint8', 'int64'), + name="train_reader" if is_train else "test_reader", + use_double_buffer=True) + input, label = fluid.layers.read_file(pyreader) + else: + input = fluid.layers.data( + name='data', shape=dshape, dtype='uint8') + label = fluid.layers.data( + name='label', shape=[1], dtype='int64') + + # add imagenet preprocessors + random_crop = fluid.layers.random_crop(input, dshape) + casted = fluid.layers.cast(random_crop, 'float32') + # input is HWC + trans = fluid.layers.transpose(casted, [0, 3, 1, 2]) / 255.0 + img_mean = fluid.layers.tensor.assign( + np.array([0.485, 0.456, 0.406]).astype('float32').reshape((3, 1, + 1))) + img_std = fluid.layers.tensor.assign( + np.array([0.229, 0.224, 0.225]).astype('float32').reshape((3, 1, + 1))) + h1 = fluid.layers.elementwise_sub(trans, img_mean, axis=1) + h2 = fluid.layers.elementwise_div(h1, img_std, axis=1) + + # pre_out = (trans - img_mean) / img_std + + predict = model(h2, class_dim, is_train=is_train) + cost = fluid.layers.cross_entropy(input=predict, label=label) + avg_cost = fluid.layers.mean(x=cost) + + batch_acc1 = fluid.layers.accuracy(input=predict, label=label, k=1) + batch_acc5 = fluid.layers.accuracy(input=predict, label=label, k=5) + + # configure optimize + optimizer = None + if is_train: + if args.use_lars: + lars_decay = 1.0 + else: + lars_decay = 0.0 + + total_images = 1281167 / trainer_count + + step = int(total_images / args.batch_size + 1) + epochs = [30, 60, 80, 90] + bd = [step * e for e in epochs] + base_lr = args.learning_rate + lr = [] + lr = [base_lr * (0.1**i) for i in range(len(bd) + 1)] + optimizer = fluid.optimizer.Momentum( + learning_rate=base_lr, + #learning_rate=fluid.layers.piecewise_decay( + # boundaries=bd, values=lr), + momentum=0.9, + regularization=fluid.regularizer.L2Decay(1e-4)) + optimizer.minimize(avg_cost) + + if args.memory_optimize: + fluid.memory_optimize(main_prog) + + # config readers + if not args.use_reader_op: + batched_reader = paddle.batch( + reader if args.no_random else paddle.reader.shuffle( + reader, buf_size=5120), + batch_size=args.batch_size * args.gpus, + drop_last=True) + else: + batched_reader = None + pyreader.decorate_paddle_reader( + paddle.batch( + # reader if args.no_random else paddle.reader.shuffle( + # reader, buf_size=5120), + reader, + batch_size=args.batch_size)) + + return avg_cost, optimizer, [batch_acc1, + batch_acc5], batched_reader, pyreader diff --git a/benchmark/fluid/models/se_resnext.py b/benchmark/fluid/models/se_resnext.py new file mode 100644 index 0000000000..9f887fb324 --- /dev/null +++ b/benchmark/fluid/models/se_resnext.py @@ -0,0 +1,286 @@ +# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import paddle +import paddle.fluid as fluid +import math +import os +from imagenet_reader import train, val + +__all__ = [ + "SE_ResNeXt", "SE_ResNeXt50_32x4d", "SE_ResNeXt101_32x4d", + "SE_ResNeXt152_32x4d", "get_model" +] + +train_parameters = { + "input_size": [3, 224, 224], + "input_mean": [0.485, 0.456, 0.406], + "input_std": [0.229, 0.224, 0.225], + "learning_strategy": { + "name": "piecewise_decay", + "batch_size": 256, + "epochs": [30, 60, 90], + "steps": [0.1, 0.01, 0.001, 0.0001] + } +} + + +class SE_ResNeXt(): + def __init__(self, layers=50, is_train=True): + self.params = train_parameters + self.layers = layers + self.is_train = is_train + + def net(self, input, class_dim=1000): + layers = self.layers + supported_layers = [50, 101, 152] + assert layers in supported_layers, \ + "supported layers are {} but input layer is {}".format(supported_layers, layers) + if layers == 50: + cardinality = 32 + reduction_ratio = 16 + depth = [3, 4, 6, 3] + num_filters = [128, 256, 512, 1024] + + conv = self.conv_bn_layer( + input=input, + num_filters=64, + filter_size=7, + stride=2, + act='relu') + conv = fluid.layers.pool2d( + input=conv, + pool_size=3, + pool_stride=2, + pool_padding=1, + pool_type='max') + elif layers == 101: + cardinality = 32 + reduction_ratio = 16 + depth = [3, 4, 23, 3] + num_filters = [128, 256, 512, 1024] + + conv = self.conv_bn_layer( + input=input, + num_filters=64, + filter_size=7, + stride=2, + act='relu') + conv = fluid.layers.pool2d( + input=conv, + pool_size=3, + pool_stride=2, + pool_padding=1, + pool_type='max') + elif layers == 152: + cardinality = 64 + reduction_ratio = 16 + depth = [3, 8, 36, 3] + num_filters = [128, 256, 512, 1024] + + conv = self.conv_bn_layer( + input=input, + num_filters=64, + filter_size=3, + stride=2, + act='relu') + conv = self.conv_bn_layer( + input=conv, num_filters=64, filter_size=3, stride=1, act='relu') + conv = self.conv_bn_layer( + input=conv, + num_filters=128, + filter_size=3, + stride=1, + act='relu') + conv = fluid.layers.pool2d( + input=conv, pool_size=3, pool_stride=2, pool_padding=1, \ + pool_type='max') + + for block in range(len(depth)): + for i in range(depth[block]): + conv = self.bottleneck_block( + input=conv, + num_filters=num_filters[block], + stride=2 if i == 0 and block != 0 else 1, + cardinality=cardinality, + reduction_ratio=reduction_ratio) + + pool = fluid.layers.pool2d( + input=conv, pool_size=7, pool_type='avg', global_pooling=True) + drop = fluid.layers.dropout(x=pool, dropout_prob=0.5) + stdv = 1.0 / math.sqrt(drop.shape[1] * 1.0) + out = fluid.layers.fc(input=drop, + size=class_dim, + act='softmax', + param_attr=fluid.param_attr.ParamAttr( + initializer=fluid.initializer.Uniform(-stdv, + stdv))) + return out + + def shortcut(self, input, ch_out, stride): + ch_in = input.shape[1] + if ch_in != ch_out or stride != 1: + filter_size = 1 + return self.conv_bn_layer(input, ch_out, filter_size, stride) + else: + return input + + def bottleneck_block(self, input, num_filters, stride, cardinality, + reduction_ratio): + conv0 = self.conv_bn_layer( + input=input, num_filters=num_filters, filter_size=1, act='relu') + conv1 = self.conv_bn_layer( + input=conv0, + num_filters=num_filters, + filter_size=3, + stride=stride, + groups=cardinality, + act='relu') + conv2 = self.conv_bn_layer( + input=conv1, num_filters=num_filters * 2, filter_size=1, act=None) + scale = self.squeeze_excitation( + input=conv2, + num_channels=num_filters * 2, + reduction_ratio=reduction_ratio) + + short = self.shortcut(input, num_filters * 2, stride) + + return fluid.layers.elementwise_add(x=short, y=scale, act='relu') + + def conv_bn_layer(self, + input, + num_filters, + filter_size, + stride=1, + groups=1, + act=None): + conv = fluid.layers.conv2d( + input=input, + num_filters=num_filters, + filter_size=filter_size, + stride=stride, + padding=(filter_size - 1) / 2, + groups=groups, + act=None, + bias_attr=False) + return fluid.layers.batch_norm( + input=conv, act=act, is_test=not self.is_train) + + def squeeze_excitation(self, input, num_channels, reduction_ratio): + pool = fluid.layers.pool2d( + input=input, pool_size=0, pool_type='avg', global_pooling=True) + stdv = 1.0 / math.sqrt(pool.shape[1] * 1.0) + squeeze = fluid.layers.fc(input=pool, + size=num_channels / reduction_ratio, + act='relu', + param_attr=fluid.param_attr.ParamAttr( + initializer=fluid.initializer.Uniform( + -stdv, stdv))) + stdv = 1.0 / math.sqrt(squeeze.shape[1] * 1.0) + excitation = fluid.layers.fc(input=squeeze, + size=num_channels, + act='sigmoid', + param_attr=fluid.param_attr.ParamAttr( + initializer=fluid.initializer.Uniform( + -stdv, stdv))) + scale = fluid.layers.elementwise_mul(x=input, y=excitation, axis=0) + return scale + + +def SE_ResNeXt50_32x4d(): + model = SE_ResNeXt(layers=50) + return model + + +def SE_ResNeXt101_32x4d(): + model = SE_ResNeXt(layers=101) + return model + + +def SE_ResNeXt152_32x4d(): + model = SE_ResNeXt(layers=152) + return model + + +def get_model(args, is_train, main_prog, startup_prog): + model = SE_ResNeXt(layers=50) + batched_reader = None + pyreader = None + trainer_count = int(os.getenv("PADDLE_TRAINERS")) + dshape = train_parameters["input_size"] + + with fluid.program_guard(main_prog, startup_prog): + with fluid.unique_name.guard(): + if args.use_reader_op: + pyreader = fluid.layers.py_reader( + capacity=10, + shapes=([-1] + dshape, (-1, 1)), + dtypes=('float32', 'int64'), + name="train_reader" if is_train else "test_reader", + use_double_buffer=True) + input, label = fluid.layers.read_file(pyreader) + else: + input = fluid.layers.data( + name='data', shape=dshape, dtype='float32') + label = fluid.layers.data( + name='label', shape=[1], dtype='int64') + + out = model.net(input=input) + cost = fluid.layers.cross_entropy(input=out, label=label) + avg_cost = fluid.layers.mean(x=cost) + acc_top1 = fluid.layers.accuracy(input=out, label=label, k=1) + acc_top5 = fluid.layers.accuracy(input=out, label=label, k=5) + + optimizer = None + if is_train: + if args.use_lars: + lars_decay = 1.0 + else: + lars_decay = 0.0 + + total_images = 1281167 / trainer_count + + step = int(total_images / args.batch_size + 1) + epochs = [40, 80, 100] + bd = [step * e for e in epochs] + base_lr = args.learning_rate + lr = [] + lr = [base_lr * (0.1**i) for i in range(len(bd) + 1)] + optimizer = fluid.optimizer.Momentum( + # learning_rate=base_lr, + learning_rate=fluid.layers.piecewise_decay( + boundaries=bd, values=lr), + momentum=0.9, + regularization=fluid.regularizer.L2Decay(1e-4), + LARS_weight_decay=lars_decay) + optimizer.minimize(avg_cost) + + if args.memory_optimize: + fluid.memory_optimize(main_prog) + + # config readers + if is_train: + reader = train() + else: + reader = val() + + if not args.use_reader_op: + batched_reader = paddle.batch( + reader, batch_size=args.batch_size * args.gpus, drop_last=True) + else: + pyreader.decorate_paddle_reader( + paddle.batch( + reader, batch_size=args.batch_size)) + + return avg_cost, optimizer, [acc_top1, acc_top5], batched_reader, pyreader diff --git a/benchmark/fluid/models/stacked_dynamic_lstm.py b/benchmark/fluid/models/stacked_dynamic_lstm.py index 3231542a17..f23bb59de9 100644 --- a/benchmark/fluid/models/stacked_dynamic_lstm.py +++ b/benchmark/fluid/models/stacked_dynamic_lstm.py @@ -26,7 +26,6 @@ import numpy import paddle import paddle.dataset.imdb as imdb import paddle.fluid as fluid -import paddle.batch as batch import paddle.fluid.profiler as profiler word_dict = imdb.word_dict() @@ -43,19 +42,7 @@ def crop_sentence(reader, crop_size): return __impl__ -def get_model(args): - if args.use_reader_op: - raise Exception( - "stacked_dynamic_lstm do not support reader op for now.") - lstm_size = 512 - emb_dim = 512 - crop_size = 1500 - - data = fluid.layers.data( - name="words", shape=[1], lod_level=1, dtype='int64') - sentence = fluid.layers.embedding( - input=data, size=[len(word_dict), emb_dim]) - +def lstm_net(sentence, lstm_size): sentence = fluid.layers.fc(input=sentence, size=lstm_size, act='tanh') rnn = fluid.layers.DynamicRNN() @@ -97,31 +84,47 @@ def get_model(args): last = fluid.layers.sequence_pool(rnn(), 'last') logit = fluid.layers.fc(input=last, size=2, act='softmax') - loss = fluid.layers.cross_entropy( - input=logit, - label=fluid.layers.data( - name='label', shape=[1], dtype='int64')) - loss = fluid.layers.mean(x=loss) + return logit - # add acc - batch_size_tensor = fluid.layers.create_tensor(dtype='int64') - batch_acc = fluid.layers.accuracy(input=logit, label=fluid.layers.data(name='label', \ - shape=[1], dtype='int64'), total=batch_size_tensor) - inference_program = fluid.default_main_program().clone() - with fluid.program_guard(inference_program): - inference_program = fluid.io.get_inference_program( - target_vars=[batch_acc, batch_size_tensor]) - - adam = fluid.optimizer.Adam() +def get_model(args, is_train, main_prog, startup_prog): + if args.use_reader_op: + raise Exception( + "stacked_dynamic_lstm do not support reader op for now.") + lstm_size = 512 + emb_dim = 512 + crop_size = 1500 - train_reader = batch( + with fluid.program_guard(main_prog, startup_prog): + with fluid.unique_name.guard(): + data = fluid.layers.data( + name="words", shape=[1], lod_level=1, dtype='int64') + sentence = fluid.layers.embedding( + input=data, size=[len(word_dict), emb_dim]) + logit = lstm_net(sentence, lstm_size) + loss = fluid.layers.cross_entropy( + input=logit, + label=fluid.layers.data( + name='label', shape=[1], dtype='int64')) + loss = fluid.layers.mean(x=loss) + + # add acc + batch_size_tensor = fluid.layers.create_tensor(dtype='int64') + batch_acc = fluid.layers.accuracy(input=logit, label=fluid.layers.data(name='label', \ + shape=[1], dtype='int64'), total=batch_size_tensor) + + if is_train: + adam = fluid.optimizer.Adam() + adam.minimize(loss) + + if is_train: + reader = crop_sentence(imdb.train(word_dict), crop_size) + else: + reader = crop_sentence(imdb.test(word_dict), crop_size) + + batched_reader = paddle.batch( paddle.reader.shuffle( - crop_sentence(imdb.train(word_dict), crop_size), buf_size=25000), + reader, buf_size=25000), batch_size=args.batch_size * args.gpus) - test_reader = batch( - paddle.reader.shuffle( - crop_sentence(imdb.test(word_dict), crop_size), buf_size=25000), - batch_size=args.batch_size) - return loss, inference_program, adam, train_reader, test_reader, batch_acc + return loss, adam, [batch_acc], batched_reader, None diff --git a/benchmark/fluid/models/vgg.py b/benchmark/fluid/models/vgg.py index 932601302d..cf9708d500 100644 --- a/benchmark/fluid/models/vgg.py +++ b/benchmark/fluid/models/vgg.py @@ -25,7 +25,7 @@ import functools import os -def vgg16_bn_drop(input): +def vgg16_bn_drop(input, is_train=True): def conv_block(input, num_filter, groups, dropouts): return fluid.nets.img_conv_group( input=input, @@ -46,13 +46,13 @@ def vgg16_bn_drop(input): drop = fluid.layers.dropout(x=conv5, dropout_prob=0.5) fc1 = fluid.layers.fc(input=drop, size=512, act=None) - bn = fluid.layers.batch_norm(input=fc1, act='relu') + bn = fluid.layers.batch_norm(input=fc1, act='relu', is_test=not is_train) drop2 = fluid.layers.dropout(x=bn, dropout_prob=0.5) fc2 = fluid.layers.fc(input=drop2, size=512, act=None) return fc2 -def get_model(args): +def get_model(args, is_train, main_prog, startup_prog): if args.data_set == "cifar10": classdim = 10 if args.data_format == 'NCHW': @@ -65,57 +65,56 @@ def get_model(args): data_shape = [3, 224, 224] else: data_shape = [224, 224, 3] + filelist = [ + os.path.join(args.data_path, f) for f in os.listdir(args.data_path) + ] + with fluid.program_guard(main_prog, startup_prog): + if args.use_reader_op: + data_file_handle = fluid.layers.open_files( + filenames=filelist, + shapes=[[-1] + data_shape, (-1, 1)], + lod_levels=[0, 0], + dtypes=["float32", "int64"], + thread_num=1, + pass_num=1) + data_file = fluid.layers.double_buffer( + fluid.layers.batch( + data_file_handle, batch_size=args.batch_size)) + with fluid.unique_name.guard(): + if args.use_reader_op: + images, label = fluid.layers.read_file(data_file) + else: + images = fluid.layers.data( + name='data', shape=data_shape, dtype='float32') + label = fluid.layers.data( + name='label', shape=[1], dtype='int64') + # Train program + net = vgg16_bn_drop(images, is_train=is_train) + predict = fluid.layers.fc(input=net, size=classdim, act='softmax') + cost = fluid.layers.cross_entropy(input=predict, label=label) + avg_cost = fluid.layers.mean(x=cost) - if args.use_reader_op: - filelist = [ - os.path.join(args.data_path, f) for f in os.listdir(args.data_path) - ] - data_file = fluid.layers.open_files( - filenames=filelist, - shapes=[[-1] + data_shape, (-1, 1)], - lod_levels=[0, 0], - dtypes=["float32", "int64"], - thread_num=args.gpus, - pass_num=args.pass_num) - data_file = fluid.layers.double_buffer( - fluid.layers.batch( - data_file, batch_size=args.batch_size)) - images, label = fluid.layers.read_file(data_file) - else: - images = fluid.layers.data( - name='data', shape=data_shape, dtype='float32') - label = fluid.layers.data(name='label', shape=[1], dtype='int64') - - # Train program - net = vgg16_bn_drop(images) - predict = fluid.layers.fc(input=net, size=classdim, act='softmax') - cost = fluid.layers.cross_entropy(input=predict, label=label) - avg_cost = fluid.layers.mean(x=cost) - - # Evaluator - batch_size_tensor = fluid.layers.create_tensor(dtype='int64') - batch_acc = fluid.layers.accuracy( - input=predict, label=label, total=batch_size_tensor) - - # inference program - inference_program = fluid.default_main_program().clone() - with fluid.program_guard(inference_program): - inference_program = fluid.io.get_inference_program( - target_vars=[batch_acc, batch_size_tensor]) - - # Optimization - optimizer = fluid.optimizer.Adam(learning_rate=args.learning_rate) + # Evaluator + batch_size_tensor = fluid.layers.create_tensor(dtype='int64') + batch_acc = fluid.layers.accuracy( + input=predict, label=label, total=batch_size_tensor) + # Optimization + if is_train: + optimizer = fluid.optimizer.Adam( + learning_rate=args.learning_rate) + optimizer.minimize(avg_cost) # data reader - train_reader = paddle.batch( + if is_train: + reader = paddle.dataset.cifar.train10() \ + if args.data_set == 'cifar10' else paddle.dataset.flowers.train() + else: + reader = paddle.dataset.cifar.test10() \ + if args.data_set == 'cifar10' else paddle.dataset.flowers.test() + + batched_reader = paddle.batch( paddle.reader.shuffle( - paddle.dataset.cifar.train10() - if args.data_set == 'cifar10' else paddle.dataset.flowers.train(), - buf_size=5120), + reader, buf_size=5120), batch_size=args.batch_size * args.gpus) - test_reader = paddle.batch( - paddle.dataset.cifar.test10() - if args.data_set == 'cifar10' else paddle.dataset.flowers.test(), - batch_size=args.batch_size) - return avg_cost, inference_program, optimizer, train_reader, test_reader, batch_acc + return avg_cost, optimizer, [batch_acc], batched_reader, data_file_handle diff --git a/paddle/fluid/API.spec b/paddle/fluid/API.spec index c2694144d7..ae5f30e431 100644 --- a/paddle/fluid/API.spec +++ b/paddle/fluid/API.spec @@ -66,7 +66,7 @@ paddle.fluid.InferenceTranspiler.transpile ArgSpec(args=['self', 'program', 'pla paddle.fluid.memory_optimize ArgSpec(args=['input_program', 'skip_opt_set', 'print_log', 'level'], varargs=None, keywords=None, defaults=(None, False, 0)) paddle.fluid.release_memory ArgSpec(args=['input_program', 'skip_opt_set'], varargs=None, keywords=None, defaults=(None,)) paddle.fluid.DistributeTranspilerConfig.__init__ -paddle.fluid.ParallelExecutor.__init__ ArgSpec(args=['self', 'use_cuda', 'loss_name', 'main_program', 'share_vars_from', 'exec_strategy', 'build_strategy', 'num_trainers', 'trainer_id'], varargs=None, keywords='kwargs', defaults=(None, None, None, None, None, 1, 0)) +paddle.fluid.ParallelExecutor.__init__ ArgSpec(args=['self', 'use_cuda', 'loss_name', 'main_program', 'share_vars_from', 'exec_strategy', 'build_strategy', 'num_trainers', 'trainer_id', 'scope'], varargs=None, keywords='kwargs', defaults=(None, None, None, None, None, 1, 0, None)) paddle.fluid.ParallelExecutor.run ArgSpec(args=['self', 'fetch_list', 'feed', 'feed_dict', 'return_numpy'], varargs=None, keywords=None, defaults=(None, None, True)) paddle.fluid.ExecutionStrategy.__init__ __init__(self: paddle.fluid.core.ExecutionStrategy) -> None paddle.fluid.BuildStrategy.GradientScaleStrategy.__init__ __init__(self: paddle.fluid.core.GradientScaleStrategy, arg0: int) -> None diff --git a/paddle/fluid/platform/nccl_helper.h b/paddle/fluid/platform/nccl_helper.h index cc46c88fd1..115abb98d5 100644 --- a/paddle/fluid/platform/nccl_helper.h +++ b/paddle/fluid/platform/nccl_helper.h @@ -100,14 +100,13 @@ struct NCCLContextMap { return; } std::unique_ptr comms(new ncclComm_t[order_.size()]); - // if pass nccl_id here, can assume we are doing multi node training - if (nccl_id == nullptr) { + // if num_trainers == 1, should create a new nccl id for local comms. + if (num_trainers == 1) { std::lock_guard guard(NCCLGroupGuard::NCCLMutex()); PADDLE_ENFORCE(platform::dynload::ncclCommInitAll( comms.get(), static_cast(order_.size()), order_.data())); } else { - PADDLE_ENFORCE_GT(num_trainers, 1); - // TODO(wuyi): need to ensure each node have same number of GPUs + PADDLE_ENFORCE_NOT_NULL(nccl_id); { int nranks = num_trainers * order_.size(); NCCLGroupGuard gurad; diff --git a/python/paddle/fluid/parallel_executor.py b/python/paddle/fluid/parallel_executor.py index a7765c9591..4790e0f611 100644 --- a/python/paddle/fluid/parallel_executor.py +++ b/python/paddle/fluid/parallel_executor.py @@ -43,8 +43,9 @@ class ParallelExecutor(object): num_trainers(int): If greater than 1, NCCL will be initialized with multiple rank of nodes, each node should have same number of GPUs. Distributed training will be enabled then. Default 1. - trainer_id(int: Must use together with num_trainers. trainer_id is the + trainer_id(int): Must use together with num_trainers. trainer_id is the "rank" of current node starts from 0. Default 0. + scope(Scope): scope to run with, default use fluid.global_scope(). Returns: ParallelExecutor: The initialized ParallelExecutor object. @@ -73,6 +74,7 @@ class ParallelExecutor(object): build_strategy=None, num_trainers=1, trainer_id=0, + scope=None, **kwargs): if len(kwargs) != 0: err_msg = "" @@ -131,7 +133,8 @@ class ParallelExecutor(object): main = main_program main = main if main else framework.default_main_program() - scope = executor.global_scope() + if scope == None: + scope = executor.global_scope() # FIXME(Yancey1989): it's a temporary approach to determinate the distribute # train program, call self.bcast_param() at the end of each mini-batch. self.is_dist = True if "recv" in [ From 14242eae745201db59e98e22b1daaae84a757688 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Sun, 9 Sep 2018 11:52:46 -0700 Subject: [PATCH 28/28] slightly restructure the document of dynamic nets (#13287) * slightly restructure the document [skip ci] * Update dynamic_graph.md --- doc/survey/dynamic_graph.md | 61 +++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/doc/survey/dynamic_graph.md b/doc/survey/dynamic_graph.md index d03212007a..7f62eeadff 100644 --- a/doc/survey/dynamic_graph.md +++ b/doc/survey/dynamic_graph.md @@ -2,28 +2,31 @@ ## Automatic Differentiation -A key challenge in the field of deep learning is to automatically derive the backward pass from the forward pass described algorithmically by researchers. Such a derivation, or a transformation of the forward pass program, has been long studied before the recent prosperity of deep learning in the field known as [automatic differentiation](https://arxiv.org/pdf/1502.05767.pdf). +A key challenge in deep learning is to automatically derive the backward pass given the forward pass as a program, which has been long studied in the field of [automatic differentiation](https://arxiv.org/pdf/1502.05767.pdf), or autodiff, before the prosperity of deep learning. -## The Tape +## Program Transformation v.s. Backtracking -Given the forward pass program (usually in Python in practices), there are two strategies to derive the backward pass: +Given the forward pass program, there are two strategies to derive the backward pass: -1. from the forward pass program itself, or -1. from the execution trace of the forward pass program, which is often known as the *tape*. +1. by transforming the forward pass program without executing it, or +1. by backtracking the execution process of the forward pass program. -This article surveys systems that follow the latter strategy. +This article is about the latter strategy. -## Dynamic Network +## The Tape and Dynamic Networks -When we train a deep learning model, the tape changes every iteration as the input data change, so we have to re-derive the backward pass every iteration. This is known as *dynamic network*. +We refer to the trace of the execution of the forward pass program as a *tape* [[1]](http://www.bcl.hamilton.ie/~barak/papers/toplas-reverse.pdf). When we train a deep learning model, the tape changes every iteration as the input data change, so we'd have to re-derive the backward pass, which is time-consuming, but also eases the case that the forward program includes control flows like if-else and for/while. With these control flows, the execution trace might change with iterations. Such changes are known as *dynamic networks* in the field of deep learning. -Deep learning systems that utilize the idea of dynamic network gained their popularities in recent years. This article surveys two representative systems: [PyTorch](https://pytorch.org/) and [DyNet](https://dynet.readthedocs.io/en/latest/). +## Typical Systems -## An Overview +Deep learning systems that utilize the idea of dynamic networks gained their popularities in recent years. This article surveys the following typical systems: -Both frameworks record a ‘tape’ of the computation and interpreting (or run-time compiling) a transformation of the tape played back in reverse. This tape is a different kind of entity than the original program.[[link]](http://www.bcl.hamilton.ie/~barak/papers/toplas-reverse.pdf) +- [DyNet](https://dynet.readthedocs.io/en/latest/) +- [PyTorch](https://pytorch.org/) +- Chainer +- Autograd from HIPS -Consider the following code feedforward model. +Before diving into these systems, let us pose an example forward pass program: ```python x = Variable(randn(20, 1))) @@ -35,9 +38,11 @@ loss = softmax(pred, label) loss.backward() ``` -### 1) Dynet uses List to encode the Tape +## The Representation of Tapes -During the forward execution, a list of operators, in this case `matmul`, `matmul` and `softmax`, are recorded in the tape, along with the necessary information needed to do the backward such as pointers to the inputs and outputs. Then the tape is played in reverse order at `loss.backward()`. +### DyNet: the Tape as a List + +DyNet uses a linear data structure, a list, to represent the tape. During the execution of the above example, it is a list of operators: `matmul`, `matmul`, and `softmax`. The list also includes information needed to do the backward pass, such as pointers to the inputs and outputs. Then the tape is played in reverse order at `loss.backward().`
@@ -69,9 +74,9 @@ digraph g { ![Alt text](https://g.gravizo.com/svg?digraph%20g%20{%20graph%20[%20rankdir%20=%20%22LR%22%20];%20node%20[%20fontsize%20=%20%2216%22%20shape%20=%20%22ellipse%22%20];%20edge%20[];%20%22node0%22%20[%20label%20=%20%22%3Cf0%3E%20type:%20matmul%20|%20%3Cf1%3E%20input:%20W_1,%20x%20|%20%3Cf2%3E%20output:%20h%22%20shape%20=%20%22record%22%20];%20%22node1%22%20[%20label%20=%20%22%3Cf0%3E%20type:%20matmul%20|%20%3Cf1%3E%20input:%20W_2,%20h%20|%20%3Cf2%3E%20output:%20pred%22%20shape%20=%20%22record%22%20];%20%22node2%22%20[%20label%20=%20%22%3Cf0%3E%20type:%20softmax%20|%20%3Cf1%3E%20input:%20pred,%20label%20|%20%3Cf2%3E%20output:%20loss%22%20shape%20=%20%22record%22%20];%20%22node0%22:f0%20-%3E%20%22node1%22:f0%20[%20id%20=%200%20];%20%22node1%22:f0%20-%3E%20%22node2%22:f0%20[%20id%20=%201%20];%20}) -### 2) Pytorch uses Node Graph to encode the Tape +### PyTorch: the Tape as a Graph -The graph is composed of `Variable`s and `Function`s. During the forward execution, a `Variable` records its creator function, e.g. `h.creator = matmul`. And a Function records its inputs' previous/dependent functions `prev_func` through `creator`, e.g. `matmul.prev_func = matmul1`. At `loss.backward()`, a topological sort is performed on all `prev_func`s. Then the grad op is performed by the sorted order. +The graph is composed of `Variable`s and `Function`s. During the forward execution, a `Variable` records its creator function, e.g. `h.creator = matmul`. And a Function records its inputs' previous/dependent functions `prev_func` through `creator`, e.g. `matmul.prev_func = matmul1`. At `loss.backward()`, a topological sort is performed on all `prev_func`s. Then the grad op is performed by the sorted order. Please be aware that a `Function` might have more than one `prev_func`s.
@@ -132,27 +137,22 @@ digraph g { ![Alt text](https://g.gravizo.com/svg?digraph%20g%20{%20graph%20[%20rankdir%20=%20%22LR%22%20];%20subgraph%20function%20{%20node%20[%20fontsize%20=%20%2216%22%20style%20=%20filled%20shape%20=%20%22record%22%20];%20%22matmul0%22%20[%20label%20=%20%22%3Cf0%3E%20type:%20matmul%20|%20prev_func:%20None%22%20];%20%22matmul1%22%20[%20label%20=%20%22%3Cf0%3E%20type:%20matmul%20|%20prev_func:%20matmul%22%20];%20%22softmax%22%20[%20label%20=%20%22%3Cf0%3E%20type:%20softmax%20|%20prev_func:%20matmul%22%20];%20}%20subgraph%20variable%20{%20node%20[%20fontsize%20=%20%2216%22%20shape%20=%20%22Mrecord%22%20style%20=%20filled%20fillcolor%20=%20white%20];%20%22x%22%20[%20label%20=%20%22%3Cf0%3E%20x%20|%20%3Cf1%3E%20creator:%20None%22%20];%20%22label%22%20[%20label%20=%20%22%3Cf0%3E%20label%20|%20%3Cf1%3E%20creator:%20None%22%20];%20%22W_1%22%20[%20label%20=%20%22%3Cf0%3E%20W_1%20|%20%3Cf1%3E%20creator:%20None%22%20];%20%22W_2%22%20[%20label%20=%20%22%3Cf0%3E%20W_2%20|%20%3Cf1%3E%20creator:%20None%22%20];%20%22h%22%20[%20label%20=%20%22%3Cf0%3E%20h%20|%20%3Cf1%3E%20creator:%20None%22%20];%20%22pred%22%20[%20label%20=%20%22%3Cf0%3E%20pred%20|%20%3Cf1%3E%20creator:%20matmul%22%20];%20%22loss%22%20[%20label%20=%20%22%3Cf0%3E%20loss%20|%20%3Cf1%3E%20creator:%20softmax%22%20];%20}%20subgraph%20data_flow%20{%20%22x%22:f0%20-%3E%20%22matmul0%22:f0;%20%22W_1%22:f0%20-%3E%20%22matmul0%22:f0;%20%22matmul0%22:f0%20-%3E%20%22h%22:f0;%20%22h%22:f0%20-%3E%20%22matmul1%22:f0;%20%22W_2%22:f0%20-%3E%20%22matmul1%22:f0;%20%22matmul1%22:f0%20-%3E%20%22pred%22:f0;%20%22pred%22:f0%20-%3E%20%22softmax%22:f0;%20%22label%22:f0%20-%3E%20%22softmax%22:f0;%20%22softmax%22:f0%20-%3E%20%22loss%22:f0;%20}%20subgraph%20prev_func%20{%20edge%20[color=%22red%22,%20arrowsize=%220.6%22,%20penwidth=%221%22,%20constraint=false];%20%22matmul1%22:f1%20-%3E%20%22matmul0%22:f0;%20%22softmax%22:f1%20-%3E%20%22matmul1%22:f0;%20label%20=%20%22prev_func%22;%20}%20}) -Chainer and Autograd uses the similar techniques to record the forward pass. For details please refer to the appendix. - -## Design choices +Chainer and Autograd use the similar techniques to record the forward pass. For details, please refer to the appendix. -### 1) Dynet's List vs Pytorch's Node Graph +## Comparison: List v.s. Graph -What's good about List: -1. It avoids a topological sort. One only needs to traverse the list of operators in reverse and calling the corresponding backward operator. -1. It promises effient data parallelism implementations. One could count the time of usage of a certain variable during the construction list. Then in the play back, one knows the calculation of a variable has completed. This enables communication and computation overlapping. +The list of DyNet could be considered the result of the topological sort of the graph of PyTorch. Or, the graph is the raw representation of the tape, which gives us the chance to *prune* part of the graph that is irrelevant with the backward pass before the topological sort [[2]](https://openreview.net/pdf?id=BJJsrmfCZ). Consider the following example, PyTorch only does backward on `SmallNet` while DyNet does both `SmallNet` and `BigNet`: -What's good about Node Graph: -1. More flexibility. PyTorch users can mix and match independent graphs however they like, in whatever threads they like (without explicit synchronization). An added benefit of structuring graphs this way is that when a portion of the graph becomes dead, it is automatically freed. [[2]](https://openreview.net/pdf?id=BJJsrmfCZ) Consider the following example, Pytorch only does backward on SmallNet while Dynet does both BigNet and SmallNet. ```python result = BigNet(data) loss = SmallNet(data) loss.backward() ``` -### 2) Dynet's Lazy evaluation vs Pytorch's Immediate evaluation +## Lazy v.s. Immediate Evaluation + +Another difference between DyNet and PyTorch is that DyNet lazily evaluates the forward pass, whereas PyTorch executes it immediately. Consider the following example: -Dynet builds the list in a symbolic matter. Consider the following example ```python for epoch in range(num_epochs): for in_words, out_label in training_data: @@ -164,16 +164,17 @@ for epoch in range(num_epochs): loss_val = loss_sym.value() loss_sym.backward() ``` + The computation of `lookup`, `concat`, `matmul` and `softmax` didn't happen until the call of `loss_sym.value()`. This defered execution is useful because it allows some graph-like optimization possible, e.g. kernel fusion. -Pytorch chooses immediate evaluation. It avoids ever materializing a "forward graph"/"tape" (no need to explicitly call `dy.renew_cg()` to reset the list), recording only what is necessary to differentiate the computation, i.e. `creator` and `prev_func`. +PyTorch chooses immediate evaluation. It avoids ever materializing a "forward graph"/"tape" (no need to explicitly call `dy.renew_cg()` to reset the list), recording only what is necessary to differentiate the computation, i.e. `creator` and `prev_func`. -## What can fluid learn from them? +## Fluid: Learning the Lessons Please refer to `paddle/contrib/dynamic/`. -# Appendix +## Appendix ### Overview