Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into fix-dist-sparse-decay
test=developrevert-15207-remove_op_handle_lock_and_fix_var
Qiao Longfei
6 years ago
commit
d0e3b24002
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,130 @@
|
||||
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef PADDLE_FLUID_FRAMEWORK_IR_LOCK_FREE_OPTIMIZE_PASS_H_
|
||||
#define PADDLE_FLUID_FRAMEWORK_IR_LOCK_FREE_OPTIMIZE_PASS_H_
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include <boost/algorithm/string/predicate.hpp>
|
||||
|
||||
#include "paddle/fluid/framework/ir/graph.h"
|
||||
#include "paddle/fluid/framework/ir/pass.h"
|
||||
|
||||
namespace paddle {
|
||||
namespace framework {
|
||||
namespace ir {
|
||||
|
||||
class Node;
|
||||
|
||||
/*
|
||||
* Remove the sum op of all gradients of the backward op.
|
||||
* And remove the dependecies of the optimizer related to the
|
||||
* same backward op.
|
||||
*
|
||||
* Before this pass:
|
||||
*
|
||||
* forward_op1 forward_op2
|
||||
* | |
|
||||
* grad_op1 grad_op2
|
||||
* \ /
|
||||
* \ /
|
||||
* sum_op
|
||||
* |
|
||||
* sgd_op
|
||||
*
|
||||
* After this pass:
|
||||
* forward_op1 forward_op2
|
||||
* | |
|
||||
* grad_op1 grad_op2
|
||||
* | |
|
||||
* sgd_op1 sgd_op2
|
||||
*
|
||||
* sgd_op1 and sgd_op2 will update the same weight which holds the same
|
||||
* memory, so we could benefits from the acceleration
|
||||
*/
|
||||
class LockFreeOptimizePass : public Pass {
|
||||
public:
|
||||
virtual ~LockFreeOptimizePass() {}
|
||||
|
||||
protected:
|
||||
std::unique_ptr<ir::Graph> ApplyImpl(std::unique_ptr<ir::Graph> graph) const;
|
||||
|
||||
private:
|
||||
// Create a new sgd node via current optimizer node
|
||||
ir::Node* CreateNewSGDNode(ir::Graph* graph, ir::Node* forward_node,
|
||||
ir::Node* backward_node, ir::Node* grad_sum_node,
|
||||
ir::Node* optimize_node) const;
|
||||
|
||||
// Replace the input weight's optimizers
|
||||
void ReplaceUpstreamNode(ir::Node* upstream_node,
|
||||
ir::Node* old_optimizer_node,
|
||||
ir::Node* new_optimizer_node) const;
|
||||
|
||||
// Replace the output weight's optimizers
|
||||
void ReplaceAllDownstreamNode(ir::Node* old_optimizer_node,
|
||||
ir::Node* new_optimizer_node) const;
|
||||
|
||||
// Find all weight variables in graph
|
||||
bool FindAllWeightVars(ir::Graph* graph) const;
|
||||
|
||||
// Find the forward_op node via the backward_op node
|
||||
ir::Node* FindForwardOpViaBackwardOp(ir::Graph* graph,
|
||||
ir::Node* backward_node) const;
|
||||
|
||||
std::vector<ir::Node*> FindConnectedNode(ir::Node* upstream_node,
|
||||
ir::Node* downstream_node) const;
|
||||
|
||||
inline bool IsOpNamed(ir::Node* node, const std::string& name) const {
|
||||
PADDLE_ENFORCE(node);
|
||||
|
||||
return node->NodeType() == Node::Type::kOperation && node->Name() == name;
|
||||
}
|
||||
|
||||
inline bool IsVarNamed(ir::Node* node, const std::string& name) const {
|
||||
PADDLE_ENFORCE(node);
|
||||
|
||||
return node->NodeType() == Node::Type::kVariable && node->Name() == name;
|
||||
}
|
||||
|
||||
inline bool IsVarNameEndsWith(ir::Node* node, const std::string& name) const {
|
||||
PADDLE_ENFORCE(node);
|
||||
|
||||
return node->NodeType() == Node::Type::kVariable &&
|
||||
boost::algorithm::ends_with(node->Name(), name);
|
||||
}
|
||||
|
||||
inline bool IsVarNameContains(ir::Node* node, const std::string& name) const {
|
||||
PADDLE_ENFORCE(node);
|
||||
|
||||
return node->NodeType() == Node::Type::kVariable &&
|
||||
node->Name().find(name) != std::string::npos;
|
||||
}
|
||||
|
||||
inline bool IsControlDepFrom(ir::Node* ctrl_dep_node, ir::Node* node) const {
|
||||
PADDLE_ENFORCE(ctrl_dep_node);
|
||||
PADDLE_ENFORCE(node);
|
||||
|
||||
return IsControlDepVar(*ctrl_dep_node) &&
|
||||
ctrl_dep_node->inputs.size() >= 1u &&
|
||||
ctrl_dep_node->inputs[0] == node;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace ir
|
||||
} // namespace framework
|
||||
} // namespace paddle
|
||||
|
||||
#endif // PADDLE_FLUID_FRAMEWORK_IR_LOCK_FREE_OPTIMIZE_PASS_H_
|
||||
@ -0,0 +1,85 @@
|
||||
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License. */
|
||||
|
||||
#include "paddle/fluid/operators/jit/gen/seqpool.h"
|
||||
#include "paddle/fluid/operators/jit/gen/act.h" // for exp_float_consts ones
|
||||
#include "paddle/fluid/operators/jit/registry.h"
|
||||
#include "paddle/fluid/platform/cpu_info.h"
|
||||
|
||||
namespace paddle {
|
||||
namespace operators {
|
||||
namespace jit {
|
||||
namespace gen {
|
||||
|
||||
void SeqPoolJitCode::genCode() {
|
||||
constexpr int block = YMM_FLOAT_BLOCK;
|
||||
constexpr int max_num_regs = 8;
|
||||
const int num_block = w_ / block;
|
||||
const int num_groups = num_block / max_num_regs;
|
||||
int rest_num_regs = num_block % max_num_regs;
|
||||
mov(reg32_int_h, dword[param_attr]);
|
||||
if (type_ == SeqPoolType::kAvg || type_ == SeqPoolType::kSqrt) {
|
||||
mov(reg_tmp, reinterpret_cast<size_t>(exp_float_consts));
|
||||
vmovups(xmm_t(1), ptr[reg_tmp + OFFSET_EXP_ONE]);
|
||||
mov(reg_tmp, reinterpret_cast<size_t>(fp_h_));
|
||||
fild(dword[param_attr]);
|
||||
fstp(dword[reg_tmp]);
|
||||
vmovss(xmm_t(0), ptr[reg_tmp]);
|
||||
if (type_ == SeqPoolType::kSqrt) {
|
||||
vsqrtps(xmm_t(0), xmm_t(0));
|
||||
}
|
||||
vdivps(xmm_t(1), xmm_t(1), xmm_t(0));
|
||||
vmovss(ptr[reg_tmp], xmm_t(1));
|
||||
}
|
||||
const int group_len = max_num_regs * block * sizeof(float);
|
||||
for (int g = 0; g < num_groups; ++g) {
|
||||
pool_height<ymm_t>(g * group_len, block, max_num_regs);
|
||||
}
|
||||
if (rest_num_regs > 0) {
|
||||
pool_height<ymm_t>(num_groups * group_len, block, rest_num_regs);
|
||||
}
|
||||
// part of rest_w * height
|
||||
const int rest = w_ % block;
|
||||
pool_height_of_rest_width(rest, (w_ - rest) * sizeof(float), max_num_regs);
|
||||
ret();
|
||||
}
|
||||
|
||||
class SeqPoolCreator : public JitCodeCreator<seq_pool_attr_t> {
|
||||
public:
|
||||
bool UseMe(const seq_pool_attr_t& attr) const override {
|
||||
return platform::MayIUse(platform::avx);
|
||||
}
|
||||
size_t CodeSize(const seq_pool_attr_t& attr) const override {
|
||||
return 96 +
|
||||
((attr.w / YMM_FLOAT_BLOCK + 4 /* for rest */) *
|
||||
4 /* load, mul and save */ +
|
||||
256) *
|
||||
8;
|
||||
}
|
||||
std::unique_ptr<GenBase> CreateJitCode(
|
||||
const seq_pool_attr_t& attr) const override {
|
||||
PADDLE_ENFORCE_GT(attr.w, 0);
|
||||
PADDLE_ENFORCE_GT(attr.h, 0);
|
||||
return make_unique<SeqPoolJitCode>(attr, CodeSize(attr));
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace gen
|
||||
} // namespace jit
|
||||
} // namespace operators
|
||||
} // namespace paddle
|
||||
|
||||
namespace gen = paddle::operators::jit::gen;
|
||||
|
||||
REGISTER_JITKERNEL_GEN(kSeqPool, gen::SeqPoolCreator);
|
||||
@ -0,0 +1,214 @@
|
||||
/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License. */
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
#include "glog/logging.h"
|
||||
#include "paddle/fluid/operators/jit/gen/jitcode.h"
|
||||
#include "paddle/fluid/platform/enforce.h"
|
||||
|
||||
namespace paddle {
|
||||
namespace operators {
|
||||
namespace jit {
|
||||
namespace gen {
|
||||
|
||||
class SeqPoolJitCode : public JitCode {
|
||||
public:
|
||||
explicit SeqPoolJitCode(const seq_pool_attr_t& attr,
|
||||
size_t code_size = 256 * 1024,
|
||||
void* code_ptr = nullptr)
|
||||
: JitCode(code_size, code_ptr), w_(attr.w), type_(attr.type) {
|
||||
if (!(type_ == SeqPoolType::kSum || type_ == SeqPoolType::kAvg ||
|
||||
type_ == SeqPoolType::kSqrt)) {
|
||||
LOG(FATAL) << "Only support sum pool yet ";
|
||||
}
|
||||
fp_h_[0] = 1.f;
|
||||
this->genCode();
|
||||
}
|
||||
|
||||
virtual const char* name() const {
|
||||
std::string base = "SeqPoolJitCode";
|
||||
if (type_ == SeqPoolType::kSum) {
|
||||
base += "_Sum";
|
||||
} else if (type_ == SeqPoolType::kAvg) {
|
||||
base += "_Avg";
|
||||
} else if (type_ == SeqPoolType::kSqrt) {
|
||||
base += "_Sqrt";
|
||||
}
|
||||
base += ("_W" + std::to_string(w_));
|
||||
return base.c_str();
|
||||
}
|
||||
void genCode() override;
|
||||
|
||||
protected:
|
||||
template <typename JMM>
|
||||
void pool_height(int w_offset, int block, int max_num_regs) {
|
||||
int offset = w_offset;
|
||||
for (int i = 0; i < max_num_regs; ++i) {
|
||||
vmovups(JMM(i), ptr[param_src + offset]);
|
||||
offset += sizeof(float) * block;
|
||||
}
|
||||
cmp(reg32_int_h, 1);
|
||||
Label l_next_h, l_h_done;
|
||||
jle(l_h_done, T_NEAR);
|
||||
mov(reg_h_i, 1);
|
||||
mov(reg_tmp, param_src);
|
||||
add(reg_tmp, w_ * sizeof(float) + w_offset);
|
||||
L(l_next_h);
|
||||
{
|
||||
mov(reg_ptr_src_i, reg_tmp);
|
||||
for (int i = 0; i < max_num_regs; ++i) {
|
||||
vmovups(JMM(i + max_num_regs), ptr[reg_ptr_src_i]);
|
||||
// sum anyway
|
||||
vaddps(JMM(i), JMM(i), JMM(i + max_num_regs));
|
||||
add(reg_ptr_src_i, sizeof(float) * block);
|
||||
}
|
||||
inc(reg_h_i);
|
||||
add(reg_tmp, w_ * sizeof(float));
|
||||
cmp(reg_h_i, reg32_int_h);
|
||||
jl(l_next_h, T_NEAR);
|
||||
}
|
||||
L(l_h_done);
|
||||
// save right now
|
||||
if (type_ == SeqPoolType::kAvg || type_ == SeqPoolType::kSqrt) {
|
||||
mov(reg_tmp, reinterpret_cast<size_t>(fp_h_));
|
||||
vbroadcastss(JMM(max_num_regs), ptr[reg_tmp]);
|
||||
}
|
||||
offset = w_offset;
|
||||
for (int i = 0; i < max_num_regs; ++i) {
|
||||
if (type_ == SeqPoolType::kAvg || type_ == SeqPoolType::kSqrt) {
|
||||
vmulps(JMM(i), JMM(i), JMM(max_num_regs));
|
||||
}
|
||||
vmovups(ptr[param_dst + offset], JMM(i));
|
||||
offset += sizeof(float) * block;
|
||||
}
|
||||
}
|
||||
|
||||
void pool_height_of_rest_width(int rest, int w_offset, int max_num_regs) {
|
||||
const int rest_used_num_regs = load_rest(rest, w_offset, 0);
|
||||
const bool has_block4 = rest / 4 > 0;
|
||||
const bool has_block2 = (rest % 4) / 2 > 0;
|
||||
const bool has_block1 = (rest % 2) == 1;
|
||||
cmp(reg32_int_h, 1);
|
||||
Label l_next_h, l_h_done;
|
||||
jle(l_h_done, T_NEAR);
|
||||
mov(reg_h_i, 1);
|
||||
mov(reg_tmp, param_src);
|
||||
add(reg_tmp, w_ * sizeof(float) + w_offset);
|
||||
L(l_next_h);
|
||||
{
|
||||
int reg_idx = 0;
|
||||
mov(reg_ptr_src_i, reg_tmp);
|
||||
if (has_block4) {
|
||||
vmovups(xmm_t(reg_idx + max_num_regs), ptr[reg_ptr_src_i]);
|
||||
add(reg_ptr_src_i, sizeof(float) * 4);
|
||||
reg_idx++;
|
||||
}
|
||||
if (has_block2) {
|
||||
vmovups(xmm_t(reg_idx + max_num_regs), ptr[reg_ptr_src_i]);
|
||||
add(reg_ptr_src_i, sizeof(float) * 2);
|
||||
reg_idx++;
|
||||
}
|
||||
if (has_block1) {
|
||||
vmovss(xmm_t(reg_idx + max_num_regs), ptr[reg_ptr_src_i]);
|
||||
reg_idx++;
|
||||
}
|
||||
PADDLE_ENFORCE_EQ(reg_idx, rest_used_num_regs,
|
||||
"All heights should use same regs");
|
||||
for (int i = 0; i < reg_idx; ++i) {
|
||||
vaddps(xmm_t(i), xmm_t(i), xmm_t(i + max_num_regs));
|
||||
}
|
||||
inc(reg_h_i);
|
||||
add(reg_tmp, w_ * sizeof(float));
|
||||
cmp(reg_h_i, reg32_int_h);
|
||||
jl(l_next_h, T_NEAR);
|
||||
}
|
||||
L(l_h_done);
|
||||
// save right now
|
||||
if (type_ == SeqPoolType::kAvg || type_ == SeqPoolType::kSqrt) {
|
||||
mov(reg_tmp, reinterpret_cast<size_t>(fp_h_));
|
||||
vbroadcastss(xmm_t(max_num_regs), ptr[reg_tmp]);
|
||||
for (int i = 0; i < rest_used_num_regs; ++i) {
|
||||
vmulps(xmm_t(i), xmm_t(i), xmm_t(max_num_regs));
|
||||
}
|
||||
}
|
||||
save_rest(rest, w_offset);
|
||||
}
|
||||
|
||||
// return the number of used regs, use start from reg 0
|
||||
int load_rest(int rest, int w_offset, const int num_shift_regs,
|
||||
const int reg_start = 0) {
|
||||
const bool has_block4 = rest / 4 > 0;
|
||||
const bool has_block2 = (rest % 4) / 2 > 0;
|
||||
const bool has_block1 = (rest % 2) == 1;
|
||||
int reg_idx = reg_start;
|
||||
if (has_block4) {
|
||||
vmovups(xmm_t(reg_idx + num_shift_regs), ptr[param_src + w_offset]);
|
||||
w_offset += sizeof(float) * 4;
|
||||
reg_idx++;
|
||||
}
|
||||
if (has_block2) {
|
||||
vmovq(xmm_t(reg_idx + num_shift_regs), ptr[param_src + w_offset]);
|
||||
w_offset += sizeof(float) * 2;
|
||||
reg_idx++;
|
||||
}
|
||||
if (has_block1) {
|
||||
vmovss(xmm_t(reg_idx + num_shift_regs), ptr[param_src + w_offset]);
|
||||
reg_idx++;
|
||||
}
|
||||
return reg_idx;
|
||||
}
|
||||
|
||||
// use reg start from 0
|
||||
void save_rest(int rest, int w_offset, int reg_start = 0) {
|
||||
const bool has_block4 = rest / 4 > 0;
|
||||
const bool has_block2 = (rest % 4) / 2 > 0;
|
||||
const bool has_block1 = (rest % 2) == 1;
|
||||
int reg_idx = reg_start;
|
||||
if (has_block4) {
|
||||
vmovups(ptr[param_dst + w_offset], xmm_t(reg_idx));
|
||||
w_offset += sizeof(float) * 4;
|
||||
reg_idx++;
|
||||
}
|
||||
if (has_block2) {
|
||||
vmovq(ptr[param_dst + w_offset], xmm_t(reg_idx));
|
||||
w_offset += sizeof(float) * 2;
|
||||
reg_idx++;
|
||||
}
|
||||
if (has_block1) {
|
||||
vmovss(ptr[param_dst + w_offset], xmm_t(reg_idx));
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
float ALIGN32_BEG fp_h_[1] ALIGN32_END;
|
||||
int w_;
|
||||
SeqPoolType type_;
|
||||
reg64_t param_src{abi_param1};
|
||||
reg64_t param_dst{abi_param2};
|
||||
reg64_t param_attr{abi_param3};
|
||||
reg64_t reg_tmp{rax};
|
||||
|
||||
reg32_t reg32_int_h{r8d};
|
||||
reg32_t reg32_fp_h{r9d};
|
||||
|
||||
reg64_t reg_h_i{r10};
|
||||
reg64_t reg_ptr_src_i{r11};
|
||||
};
|
||||
|
||||
} // namespace gen
|
||||
} // namespace jit
|
||||
} // namespace operators
|
||||
} // namespace paddle
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue