Add bn and relu fuse pass (#22048)
* add bn and relu fuse pass * add op attr assert and dtype assert * fix some inputs&&outputs bugs for the fused op and pattern. * add the unittest for fuse_bn_act_pass. test=develop * use normative enforce statements. test=develop * add the cpu test. test=develop * add the support of batch_size=1 for the bn with relu op. test=develop * add the error type for paddle throws. test=develop * add fused_batch_norm_act and fused_batch_norm_act_grad to op_has_unsed_vars_white_list. test=developrevert-22710-feature/integrated_ps_api
Zhen Wang
6 years ago
committed by
Zeng Jinle
parent
0d82baf837
commit
46189b166d
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,64 @@
|
|||||||
|
// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include <string>
|
||||||
|
#include <unordered_set>
|
||||||
|
#include <utility>
|
||||||
|
#include <vector>
|
||||||
|
#include "paddle/fluid/framework/ir/fuse_pass_base.h"
|
||||||
|
#include "paddle/fluid/framework/ir/graph.h"
|
||||||
|
#include "paddle/fluid/framework/ir/graph_pattern_detector.h"
|
||||||
|
#include "paddle/fluid/framework/ir/pass.h"
|
||||||
|
|
||||||
|
namespace paddle {
|
||||||
|
namespace framework {
|
||||||
|
namespace ir {
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Fuse the BatchNorm and activation.
|
||||||
|
*/
|
||||||
|
class FuseBatchNormActPass : public FusePassBase {
|
||||||
|
public:
|
||||||
|
virtual ~FuseBatchNormActPass() {}
|
||||||
|
|
||||||
|
protected:
|
||||||
|
void ApplyImpl(ir::Graph *graph) const override;
|
||||||
|
|
||||||
|
ir::Graph *FuseBatchNormAct(
|
||||||
|
ir::Graph *graph, const std::unordered_set<std::string> &act_types) const;
|
||||||
|
|
||||||
|
ir::Graph *FuseBatchNormActGrad(
|
||||||
|
ir::Graph *graph,
|
||||||
|
const std::unordered_set<std::string> &act_grad_types) const;
|
||||||
|
|
||||||
|
std::vector<Node *> ReplaceNode(Node *cur_node, Node *new_node,
|
||||||
|
const std::vector<Node *> &nodes) const;
|
||||||
|
|
||||||
|
void ReLinkNodes(Graph *graph, const Node *intermediate_out, Node *op_1,
|
||||||
|
Node *op_2, Node *fused_op) const;
|
||||||
|
Node *CreateFusedBatchNormActNode(
|
||||||
|
Graph *g, const Node *act, const Node *bn, const std::string &bn_x_n,
|
||||||
|
const std::string &bn_scale_n, const std::string &bn_bias_n,
|
||||||
|
const std::string &bn_variance_n, const std::string &bn_mean_n,
|
||||||
|
const std::string &bn_mean_out_n, const std::string &bn_variance_out_n,
|
||||||
|
const std::string &bn_saved_variance_n,
|
||||||
|
const std::string &bn_saved_mean_n, const std::string &bn_reserve_space_n,
|
||||||
|
const std::string &act_out_n) const;
|
||||||
|
};
|
||||||
|
|
||||||
|
} // namespace ir
|
||||||
|
} // namespace framework
|
||||||
|
} // namespace paddle
|
||||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,109 @@
|
|||||||
|
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License. */
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include <memory>
|
||||||
|
#include <string>
|
||||||
|
#include <unordered_map>
|
||||||
|
#include "paddle/fluid/framework/grad_op_desc_maker.h"
|
||||||
|
#include "paddle/fluid/framework/op_proto_maker.h"
|
||||||
|
#include "paddle/fluid/framework/operator.h"
|
||||||
|
#include "paddle/fluid/framework/tensor.h"
|
||||||
|
|
||||||
|
namespace paddle {
|
||||||
|
namespace operators {
|
||||||
|
using Tensor = framework::Tensor;
|
||||||
|
|
||||||
|
class FusedBatchNormActOp : public framework::OperatorWithKernel {
|
||||||
|
public:
|
||||||
|
using framework::OperatorWithKernel::OperatorWithKernel;
|
||||||
|
void InferShape(framework::InferShapeContext* ctx) const override;
|
||||||
|
|
||||||
|
protected:
|
||||||
|
framework::OpKernelType GetExpectedKernelType(
|
||||||
|
const framework::ExecutionContext& ctx) const override;
|
||||||
|
|
||||||
|
framework::OpKernelType GetKernelTypeForVar(
|
||||||
|
const std::string& var_name, const Tensor& tensor,
|
||||||
|
const framework::OpKernelType& expected_kernel_type) const override;
|
||||||
|
};
|
||||||
|
|
||||||
|
class FusedBatchNormActGradOp : public framework::OperatorWithKernel {
|
||||||
|
public:
|
||||||
|
using framework::OperatorWithKernel::OperatorWithKernel;
|
||||||
|
void InferShape(framework::InferShapeContext* ctx) const override;
|
||||||
|
|
||||||
|
protected:
|
||||||
|
framework::OpKernelType GetExpectedKernelType(
|
||||||
|
const framework::ExecutionContext& ctx) const override;
|
||||||
|
};
|
||||||
|
|
||||||
|
class FusedBatchNormActOpMaker : public framework::OpProtoAndCheckerMaker {
|
||||||
|
public:
|
||||||
|
void Make() override;
|
||||||
|
};
|
||||||
|
|
||||||
|
template <typename T>
|
||||||
|
class FusedBatchNormActGradOpMaker : public framework::SingleGradOpMaker<T> {
|
||||||
|
public:
|
||||||
|
using framework::SingleGradOpMaker<T>::SingleGradOpMaker;
|
||||||
|
|
||||||
|
protected:
|
||||||
|
std::unique_ptr<T> Apply() const override {
|
||||||
|
std::unique_ptr<T> op(new T());
|
||||||
|
op->SetType(this->ForwardOpType() + "_grad");
|
||||||
|
op->SetInput("X", this->Input("X"));
|
||||||
|
op->SetInput("Y", this->Output("Y"));
|
||||||
|
op->SetInput(framework::GradVarName("Y"), this->OutputGrad("Y"));
|
||||||
|
|
||||||
|
op->SetInput("Scale", this->Input("Scale"));
|
||||||
|
op->SetInput("Bias", this->Input("Bias"));
|
||||||
|
op->SetInput("SavedMean", this->Output("SavedMean"));
|
||||||
|
op->SetInput("SavedVariance", this->Output("SavedVariance"));
|
||||||
|
op->SetInput("ReserveSpace", this->Output("ReserveSpace"));
|
||||||
|
|
||||||
|
op->SetAttrMap(this->Attrs());
|
||||||
|
|
||||||
|
op->SetOutput(framework::GradVarName("X"), this->InputGrad("X"));
|
||||||
|
op->SetOutput(framework::GradVarName("Scale"), this->InputGrad("Scale"));
|
||||||
|
op->SetOutput(framework::GradVarName("Bias"), this->InputGrad("Bias"));
|
||||||
|
|
||||||
|
return op;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
class FusedBatchNormActOpInferVarType
|
||||||
|
: public framework::PassInDtypeAndVarTypeToOutput {
|
||||||
|
protected:
|
||||||
|
std::unordered_map<std::string, std::string> GetInputOutputWithSameType()
|
||||||
|
const override {
|
||||||
|
return std::unordered_map<std::string, std::string>{{"X", /*->*/ "Y"}};
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
template <typename DeviceContext, typename T>
|
||||||
|
class FusedBatchNormActKernel : public framework::OpKernel<T> {
|
||||||
|
public:
|
||||||
|
void Compute(const framework::ExecutionContext& ctx) const override;
|
||||||
|
};
|
||||||
|
|
||||||
|
template <typename DeviceContext, typename T>
|
||||||
|
class FusedBatchNormActGradKernel : public framework::OpKernel<T> {
|
||||||
|
public:
|
||||||
|
void Compute(const framework::ExecutionContext& ctx) const override;
|
||||||
|
};
|
||||||
|
|
||||||
|
} // namespace operators
|
||||||
|
} // namespace paddle
|
||||||
@ -0,0 +1,121 @@
|
|||||||
|
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
import paddle
|
||||||
|
import paddle.fluid as fluid
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
|
||||||
|
class TestFuseBatchNormActPass(unittest.TestCase):
|
||||||
|
def build_program(self, main_program, startup_program, use_cuda, seed=1):
|
||||||
|
main_program.random_seed = seed
|
||||||
|
startup_program.random_seed = seed
|
||||||
|
with fluid.program_guard(main_program, startup_program):
|
||||||
|
x = fluid.layers.data(name='x', shape=[1, 28, 28], dtype='float32')
|
||||||
|
y = fluid.layers.data(name="y", shape=[1], dtype='int64')
|
||||||
|
hidden1 = fluid.layers.conv2d(
|
||||||
|
input=x,
|
||||||
|
filter_size=3,
|
||||||
|
num_filters=32,
|
||||||
|
stride=1,
|
||||||
|
padding=1,
|
||||||
|
act=None,
|
||||||
|
bias_attr=False,
|
||||||
|
data_format='NHWC')
|
||||||
|
param_attr = fluid.ParamAttr(
|
||||||
|
name='batch_norm_w',
|
||||||
|
initializer=fluid.initializer.Constant(value=1.0))
|
||||||
|
bias_attr = fluid.ParamAttr(
|
||||||
|
name='batch_norm_b',
|
||||||
|
initializer=fluid.initializer.Constant(value=0.0))
|
||||||
|
hidden2 = fluid.layers.batch_norm(
|
||||||
|
input=hidden1,
|
||||||
|
param_attr=param_attr,
|
||||||
|
bias_attr=bias_attr,
|
||||||
|
act='relu',
|
||||||
|
data_layout='NHWC')
|
||||||
|
hidden3 = fluid.layers.fc(input=hidden2, size=128, act='relu')
|
||||||
|
hidden4 = fluid.layers.batch_norm(
|
||||||
|
input=hidden3, act='relu', data_layout='NHWC')
|
||||||
|
prediction = fluid.layers.fc(input=hidden4, size=10, act='softmax')
|
||||||
|
loss = fluid.layers.cross_entropy(input=prediction, label=y)
|
||||||
|
loss = fluid.layers.mean(loss)
|
||||||
|
sgd = fluid.optimizer.SGD(learning_rate=0.001)
|
||||||
|
if use_cuda:
|
||||||
|
sgd = fluid.contrib.mixed_precision.decorate(
|
||||||
|
sgd, use_dynamic_loss_scaling=True, init_loss_scaling=128.0)
|
||||||
|
sgd.minimize(loss)
|
||||||
|
return x, y, loss
|
||||||
|
|
||||||
|
def check(self, place, use_cuda):
|
||||||
|
main_program = fluid.Program()
|
||||||
|
startup_program = fluid.Program()
|
||||||
|
x, y, loss = self.build_program(main_program, startup_program, use_cuda)
|
||||||
|
exe = fluid.Executor(place)
|
||||||
|
iters = 10
|
||||||
|
batch_size = 16
|
||||||
|
feeder = fluid.DataFeeder(feed_list=[x, y], place=place)
|
||||||
|
|
||||||
|
# close fused_bn_act_ops
|
||||||
|
build_strategy = fluid.BuildStrategy()
|
||||||
|
build_strategy.fuse_bn_act_ops = False
|
||||||
|
binary = fluid.CompiledProgram(main_program).with_data_parallel(
|
||||||
|
loss_name=loss.name, build_strategy=build_strategy)
|
||||||
|
train_reader = paddle.batch(
|
||||||
|
paddle.dataset.mnist.train(), batch_size=batch_size)
|
||||||
|
loss_vals = []
|
||||||
|
scope = fluid.Scope()
|
||||||
|
with fluid.scope_guard(scope):
|
||||||
|
exe.run(startup_program)
|
||||||
|
for _ in range(iters):
|
||||||
|
data = next(train_reader())
|
||||||
|
loss_v = exe.run(binary,
|
||||||
|
feed=feeder.feed(data),
|
||||||
|
fetch_list=[loss])
|
||||||
|
loss_vals.append(loss_v[0][0])
|
||||||
|
|
||||||
|
# open fused_bn_act_ops
|
||||||
|
build_strategy_fused = fluid.BuildStrategy()
|
||||||
|
build_strategy_fused.fuse_bn_act_ops = True
|
||||||
|
binary_fused = fluid.CompiledProgram(main_program).with_data_parallel(
|
||||||
|
loss_name=loss.name, build_strategy=build_strategy_fused)
|
||||||
|
train_reader_fused = paddle.batch(
|
||||||
|
paddle.dataset.mnist.train(), batch_size=batch_size)
|
||||||
|
loss_vals_fused = []
|
||||||
|
scope_fused = fluid.Scope()
|
||||||
|
with fluid.scope_guard(scope_fused):
|
||||||
|
exe.run(startup_program)
|
||||||
|
for _ in range(iters):
|
||||||
|
data = next(train_reader_fused())
|
||||||
|
loss_v = exe.run(binary_fused,
|
||||||
|
feed=feeder.feed(data),
|
||||||
|
fetch_list=[loss])
|
||||||
|
loss_vals_fused.append(loss_v[0][0])
|
||||||
|
|
||||||
|
# check loss
|
||||||
|
for i in range(iters):
|
||||||
|
self.assertAlmostEqual(loss_vals[i], loss_vals_fused[i], delta=1e-5)
|
||||||
|
|
||||||
|
def test_fuse_bn_act_pass_cpu(self):
|
||||||
|
place = fluid.CPUPlace()
|
||||||
|
self.check(place, use_cuda=False)
|
||||||
|
|
||||||
|
def test_fuse_bn_act_pass_cuda(self):
|
||||||
|
if fluid.core.is_compiled_with_cuda():
|
||||||
|
place = fluid.CUDAPlace(0)
|
||||||
|
self.check(place, use_cuda=True)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
unittest.main()
|
||||||
Loading…
Reference in new issue